You are viewing a plain text version of this content. The canonical link for it is here.
Posted to alois-commits@incubator.apache.org by fl...@apache.org on 2010/11/04 18:27:42 UTC
svn commit: r1031127 [4/22] - in /incubator/alois/trunk: ./ bin/ debian/
doc/ etc/ etc/alois/ etc/alois/apache2/ etc/alois/environments/
etc/alois/prisma/ etc/cron.d/ etc/default/ etc/logrotate.d/ prisma/
prisma/bin/ prisma/conf/ prisma/conf/prisma/ pr...
Added: incubator/alois/trunk/prisma/data/prisma/prisma_database/model/postfix_detail_meta.rb
URL: http://svn.apache.org/viewvc/incubator/alois/trunk/prisma/data/prisma/prisma_database/model/postfix_detail_meta.rb?rev=1031127&view=auto
==============================================================================
--- incubator/alois/trunk/prisma/data/prisma/prisma_database/model/postfix_detail_meta.rb (added)
+++ incubator/alois/trunk/prisma/data/prisma/prisma_database/model/postfix_detail_meta.rb Thu Nov 4 18:27:22 2010
@@ -0,0 +1,54 @@
+# Copyright 2010 The Apache Software Foundation.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+ class PostfixDetailMeta < ActiveRecord::Base
+
+ description "Parsed postfix detail messages."
+ sources ["PostfixMeta"]
+
+ def self.may_have_messages?; false; end
+
+ def self.expressions
+ ret = []
+
+ ret.push({ :regex => /^size=(\d*), nrcpt=(\d*) \(([^\)]*)\)( *)$/,
+ :fields => [:size, :nrcpt, :status, nil]})
+
+ #to=<te...@mail.example.com>, orig_to=<po...@example.com>, relay=local, delay=0, status=sent (delivered to command: /usr/bin/procmail -a "$EXTENSION" DEFAULT=$HOME/Maildir/ MAILDIR=$HOME/Maildir)
+ #to=<te...@mail.example.com>, orig_to=<ad...@example.com>, relay=mail2.example.com[192.168.123.123], delay=1, status=sent (250 Ok: queued as 12AF7B902B8)
+
+ #to=<ar...@mail.logintas.ch>, orig_to=<ar...@mail.logintas.ch>, relay=local, delay=0.41, delays=0.24/0/0/0.17, dsn=2.0.0, status=sent (delivered to command: /usr/bin/procmail -a "$EXTENSION" DEFAULT=$HOME/Maildir/ MAILDIR=$HOME/Maildir)
+ ret.push({ :regex => /(orig_to=<([^>]*)>, )?relay=([^\[,]*)(\[([^\]]*)\])?(\:(\d+))?, delay=(\d+\.?\d*), (delays=(\d+\.?\d*)\/(\d+\.?\d*)\/(\d+\.?\d*)\/(\d+\.?\d*), )?(dsn=([^\,]*), )?status=([^ ]*) \((.* ([^ ]+) as ([A-F0-9]{11}))\) *$/,
+ :fields => [nil, :orig_to, :relay_host, nil, :relay_ip, nil, :relay_port, :delay,nil,:delay_before_qmgr, :delay_in_qmgr ,:delay_conn_setup, :delay_transmission, nil, :dsn, :status, :result_text, :result, :result_mail_id]})
+
+ # this is the same as above without the last .. as .. regexp (result_text only)
+ ret.push({ :regex => /(orig_to=<([^>]*)>, )?relay=([^\[,]*)(\[([^\]]*)\])?(\:(\d+))?, delay=(\d+\.?\d*), (delays=(\d+\.?\d*)\/(\d+\.?\d*)\/(\d+\.?\d*)\/(\d+\.?\d*), )?(dsn=([^\,]*), )?status=([^ ]*) \((.*)\) *$/,
+ :fields => [nil, :orig_to, :relay_host, nil,:relay_ip, nil,:relay_port, :delay,nil,:delay_before_qmgr, :delay_in_qmgr ,:delay_conn_setup, :delay_transmission, nil, :dsn, :status, :result_text]})
+
+# to=<ar...@mail.logintas.ch>, relay=127.0.0.1[127.0.0.1]:10024, delay=5, delays=0.03/0/0/4.9, dsn=2.6.0, status=sent (250 2.6.0 Ok, id=31764-06, from MTA([127.0.0.1]:10025): 250 2.0.0 Ok: queued as C1C991F0FA9)=> nil
+
+
+# ret.push({ :regex => /^to=<([^>]*)>, (orig_to=<([^>]*)>, )?relay=([^\[,]*)(\[[^\]]*\])?, delay=(\d+\.?\d*), (delays=(\d+\.?\d*)\/(\d+\.?\d*)\/(\d+\.?\d*)\/(\d+\.?\d*), )?(dsn=([^\,]*, )?status=([^ ]*) \(.*\)$/,
+# :fields => [:to, nil, :orig_to, :relay_host, :relay_ip, :delay,nil,:delay_before_qmgr, :delay_in_qmgr ,:delay_conn_setup, :delay_transmission, nil, :dsn, :status, :result_text]})
+
+
+ #to=<ar...@mail.logintas.ch>, relay=127.0.0.1[127.0.0.1]:10024, delay=5.7, delays=0.1/0/0/5.6, dsn=2.6.0, status=sent (250 2.6.0 Ok, id=15500-09, from MTA([127.0.0.1]:10025): 250 2.0.0 Ok: queued as 76B191F10AA)
+
+
+# (delivered to command: /usr/bin/procmail -a "$EXTENSION" DEFAULT=$HOME/Maildir/ MAILDIR=$HOME/Maildir)
+# (250 2.6.0 Ok, id=15500-09, from MTA([127.0.0.1]:10025): 250 2.0.0 Ok: queued as 76B191F10AA)
+ return ret
+ end
+
+ end
Added: incubator/alois/trunk/prisma/data/prisma/prisma_database/model/postfix_meta.rb
URL: http://svn.apache.org/viewvc/incubator/alois/trunk/prisma/data/prisma/prisma_database/model/postfix_meta.rb?rev=1031127&view=auto
==============================================================================
--- incubator/alois/trunk/prisma/data/prisma/prisma_database/model/postfix_meta.rb (added)
+++ incubator/alois/trunk/prisma/data/prisma/prisma_database/model/postfix_meta.rb Thu Nov 4 18:27:22 2010
@@ -0,0 +1,45 @@
+# Copyright 2010 The Apache Software Foundation.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+ class PostfixMeta < ActiveRecord::Base
+
+ description "Postfix Logs"
+ sources ["PureMeta", "LogMeta"]
+
+ preseed_expression /^postfix\//
+
+ def self.expressions
+ ret = []
+
+ ret.push({ :regex => /^postfix\/([^\[]*)\[([^\]]*)\]: (connect|disconnect) from ([^\[]*)\[([^\]]*)\]( *)$/,
+ :fields => [:program, :process_id, :action, :host, :ip, nil]})
+
+ ret.push({ :regex => /^postfix\/([^\[]*)\[([^\]]*)\]: ([A-Z0-9]*): (removed)( *)$/,
+ :fields => [:program, :process_id, :mail_message_id, :action, nil]})
+
+ ret.push({ :regex => /^postfix\/([^\[]*)\[([^\]]*)\]: ((warning):.*)$/,
+ :fields => [:program, :process_id, :message, :action]})
+
+ ret.push({ :regex => /^postfix\/([^\[]*)\[([^\]]*)\]: ([A-Z0-9]*): (client)=([^\[]*)\[([^\]]*)\]( *)$/,
+ :fields => [:program, :process_id, :mail_message_id, :action, :host, :ip, nil]})
+
+ ret.push({ :regex => /^postfix\/([^\[]*)\[([^\]]*)\]: ([A-Z0-9]*): (from|to)=<([^@]+)@([^\>]+)>, (.*)$/,
+ :fields => [:program, :process_id, :mail_message_id,:action, :email_local_part, :email_domain, :message]})
+
+ ret.push({ :regex => /^postfix\/([^\[]*)\[([^\]]*)\]: ([A-Z0-9]*): (message-id)=<([^@]+)@([^\>]+)>$/,
+ :fields => [:program, :process_id, :mail_message_id, :action, :message_id, :email_domain]})
+ return ret
+ end
+
+ end
Added: incubator/alois/trunk/prisma/data/prisma/prisma_database/model/pure_meta.rb
URL: http://svn.apache.org/viewvc/incubator/alois/trunk/prisma/data/prisma/prisma_database/model/pure_meta.rb?rev=1031127&view=auto
==============================================================================
--- incubator/alois/trunk/prisma/data/prisma/prisma_database/model/pure_meta.rb (added)
+++ incubator/alois/trunk/prisma/data/prisma/prisma_database/model/pure_meta.rb Thu Nov 4 18:27:22 2010
@@ -0,0 +1,46 @@
+# Copyright 2010 The Apache Software Foundation.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+ class PureMeta < ActiveRecord::Base
+
+ description "Meta record for a single message line"
+ sources ["FileMeta","CompressionMeta"]
+
+
+ def self.create_meta( meta_message, message)
+ return nil if CompressionMeta.applyable?(meta_message,message)
+
+ case meta_message
+ when FileMeta, CompressionMeta
+ $log.debug{"Filetype of file meta is '#{meta_message.filetype}'."}
+ if meta_message.filetype == "log" or meta_message.filetype == "syslog" then
+
+ if meta_message == FileMeta
+ # check if the message has correct size
+ throw "Message length '#{message.msg.length}' not equal to original file size '#{meta_message.size}'!" if
+ message.msg.length != meta_message.size
+ end
+ for line in message.msg
+ $log.debug("Pure Meta created:'#{line}'") if $log.debug?
+ pure = PureMeta.new.prisma_initialize(meta_message, {:message=>line})
+ pure.transform
+ end
+ end
+ meta_message.message = nil
+ return nil
+ end
+ return nil
+ end
+ end
+
Added: incubator/alois/trunk/prisma/data/prisma/prisma_database/model/source_db_meta.rb
URL: http://svn.apache.org/viewvc/incubator/alois/trunk/prisma/data/prisma/prisma_database/model/source_db_meta.rb?rev=1031127&view=auto
==============================================================================
--- incubator/alois/trunk/prisma/data/prisma/prisma_database/model/source_db_meta.rb (added)
+++ incubator/alois/trunk/prisma/data/prisma/prisma_database/model/source_db_meta.rb Thu Nov 4 18:27:22 2010
@@ -0,0 +1,220 @@
+# Copyright 2010 The Apache Software Foundation.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+ class SourceDbMeta < ActiveRecord::Base
+ description "Metadatas for database sources (pumpy raw table imports)."
+
+ def raw_class
+ return @raw_class if @raw_class
+ @raw_class = eval "#{self.raw_class_name}"
+ return @raw_class
+ end
+
+ def prisma_initialize( p_type, r_class, count=nil, current = nil, execute_once = false, waiting_time = nil )
+ if not waiting_time
+ if r_class.respond_to?(:default_waiting_time) then
+ waiting_time = r_class.default_waiting_time
+ else
+ waiting_time = $default_waiting_time || 5
+ end
+ end
+
+ if not count
+ if r_class.respond_to?(:default_count) then
+ count = r_class.default_count
+ else
+ count = $default_count || 100
+ end
+ end
+
+ self.process_type = p_type
+ self.start = current
+ self.total = 0
+ self.current = current
+ self.count = count
+ @raw_class = r_class
+ self.raw_class_name = r_class.name
+ self.execute_once = execute_once
+ self.waiting_time = waiting_time
+ self.save
+
+ @archivator = nil
+ throw "No archive pattern defined! Please specify in the config." unless $archive_pattern
+
+ @archivator = Archivator.new($archive_pattern, r_class)
+ self
+ end
+
+ def may_contain_dublettes
+ return true if raw_class.respond_to?(:may_contain_dublettes) and
+ raw_class.may_contain_dublettes
+ return false
+ end
+
+ def messages
+ skip_count = 0
+ max_count_time_per_iteration = 0.1
+
+ return nil if self.finished
+ raise LocalJumpError unless block_given?
+
+ profile = false
+ require 'ruby-prof' if profile
+ begin
+ m_table = eval "#{raw_class}.table_name"
+
+ while not $terminate
+ RubyProf.start if profile
+ case self.process_type
+ when :fifo, :firsts
+ query = "SELECT #{m_table}.* FROM #{m_table} ORDER BY id ASC LIMIT #{self.count}"
+ when :lasts
+ query = "SELECT #{m_table}.* FROM #{m_table} ORDER BY id DESC LIMIT #{self.count}"
+ # don't know if this is still correct (id > self.ucrrent?)
+ #when :from_id
+ #if self.current == nil then
+ # query = "SELECT #{m_table}.* FROM #{m_table} ORDER BY id ASC LIMIT 1"
+ #else
+ # query = "SELECT #{m_table}.* FROM #{m_table} WHERE id > #{self.current} ORDER BY ID ASC LIMIT #{self.count}"
+ #end
+ when :all
+ if self.current == nil then
+ query = "SELECT #{m_table}.* FROM #{m_table} ORDER BY id ASC LIMIT #{self.count}"
+ else
+ query = "SELECT #{m_table}.* FROM #{m_table} WHERE id > #{self.current} ORDER BY id ASC LIMIT #{self.count}"
+ end
+ end
+
+ $log.debug("Getting messages with #{query}") if $log.debug?
+ iter = raw_class.send(:find_by_sql,query)
+ loop_count = 0
+ main_start = Time.now
+
+ first = iter.first
+ last = iter.last
+
+ begin
+ Prisma::Database.transaction(self) do
+ iter.each { |raw_message|
+ loop_count += 1
+# cost = Benchmark.measure {
+ @archivator.archivate(raw_message) if @archivator
+ yield raw_message
+ self.current = raw_message.id
+# }.real
+ #p "#{cost}s: #{raw_message.inspect}"
+ if $terminate
+ $log.debug { "db_source, going to terminate."}
+ break
+ end
+ }
+ end
+ rescue ActiveRecord::Transactions::TransactionError
+ self.finished = true
+ self.save
+ raise $!
+ end
+
+ if first and last
+ Prisma::Database.transaction(raw_class) do
+ if self.process_type == :lasts
+ # ex: 100 <= id AND id <= 200
+ raw_class.delete_all "#{first.id} >= id AND id >= #{last.id}"
+ else
+ # ex: 100 <= id AND id <= 200
+ raw_class.delete_all "#{first.id} <= id AND id <= #{last.id}"
+ end
+ end
+ end
+ @archivator.close_unused_files if @archivator
+
+ main_ende = Time.now
+ main_cost = main_ende - main_start
+
+ done_one = loop_count > 0
+ if self.process_type == :all and not done_one or self.execute_once
+ self.finished=true
+ self.save
+ return
+ end
+ if not done_one then
+ Prisma::Util.perf{"No new record in table #{m_table}. Waiting #{self.waiting_time} seconds."}
+ count, rest = self.waiting_time.divmod(5)
+ count.times {
+ Prisma::Util.save_sleep(5)
+ # keep connection alive
+ raw_class.find(:first)
+ self.save
+ break if $terminate
+ }
+ Prisma::Util.save_sleep(rest)
+
+ else
+ self.start = self.current if self.start == nil
+ case self.process_type
+ when :lasts, :fifo, :firsts
+ if skip_count > 0
+ skip_count -= 1
+ Prisma::Util.perf{"Slow count, skipping todo counting for another #{skip_count} iterations"}
+ else
+ t = self.todo
+ count_time = Benchmark.measure {
+ case self.process_type
+ when :fifo, :firsts
+ self.todo = raw_class.connection.select_value("select (select id from #{raw_class.table_name} order by id desc limit 1) - (select id from #{raw_class.table_name} limit 1)").to_i
+ else
+ self.todo = raw_class.count()
+ end
+ }.real
+
+ Prisma::Util.perf{"Todo #{self.todo}, todo delta: #{self.todo - t}"} if t
+
+ if count_time > max_count_time_per_iteration
+ skip_count = (count_time / max_count_time_per_iteration).to_i
+ Prisma::Util.perf{"Counttime is #{count_time} > #{max_count_time_per_iteration} Skip counting for #{skip_count} iterations"}
+ end
+ end
+ when :from_id
+ self.todo = raw_class.count(:conditions => "id > #{self.current}")
+ end
+ self.total = self.total + loop_count
+ Prisma::Util.perf {"Done #{loop_count} in #{main_cost}s (#{loop_count / main_cost}/s)."}
+ Prisma::Util.perf {"Current record is #{self.current} done #{self.total}."}
+ end
+ self.save
+ # end
+ if profile
+ result = RubyProf.stop
+ printer = RubyProf::FlatPrinter.new(result)
+ str = StringIO.new
+ printer.print(str , :min_percent => 3)
+ str.string.split("\n").each {|l| $log.perf{l}}
+ end
+ end
+ $log.debug{"Finising source_db #{self.id}"}
+ self.finished = true
+ self.save
+ ensure
+ @archivator.close_all_files if @archivator
+ end
+ $log.info("End transform source db meta with raw class: #{raw_class_name}")
+ end
+
+ def to_s
+ "<SourceDbMeta #{total} x #{raw_class_name} (#{created_at.strftime("%F %T")})>"
+ rescue
+ super
+ end
+
+ end
Added: incubator/alois/trunk/prisma/data/prisma/prisma_database/model/squid_meta.rb
URL: http://svn.apache.org/viewvc/incubator/alois/trunk/prisma/data/prisma/prisma_database/model/squid_meta.rb?rev=1031127&view=auto
==============================================================================
--- incubator/alois/trunk/prisma/data/prisma/prisma_database/model/squid_meta.rb (added)
+++ incubator/alois/trunk/prisma/data/prisma/prisma_database/model/squid_meta.rb Thu Nov 4 18:27:22 2010
@@ -0,0 +1,165 @@
+# Copyright 2010 The Apache Software Foundation.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+class SquidMeta < ActiveRecord::Base
+ description "Parsed squid messages"
+ sources ["PureMeta", "LogMeta"]
+ def self.may_have_messages?; true; end
+
+# from: http://www.squid-cache.org/Versions/v2/2.6/cfgman/logformat.html
+# Usage:
+#
+# logformat <name> <format specification>
+#
+# Defines an access log format.
+#
+# The <format specification> is a string with embedded % format codes
+#
+# % format codes all follow the same basic structure where all but
+# the formatcode is optional. Output strings are automatically escaped
+# as required according to their context and the output format
+# modifiers are usually not needed, but can be specified if an explicit
+# output format is desired.
+#
+# % ["|[|'|#] [-] [[0]width] [{argument}] formatcode
+#
+# " output in quoted string format
+# [ output in squid text log format as used by log_mime_hdrs
+# # output in URL quoted format
+# ' output as-is
+#
+# - left aligned
+# width field width. If starting with 0 the
+# output is zero padded
+# {arg} argument such as header name etc
+#
+# Format codes:
+#
+# >a Client source IP address
+# >A Client FQDN
+# >p Client source port
+# <A Server IP address or peer name
+# la Local IP address (http_port)
+# lp Local port number (http_port)
+# ts Seconds since epoch
+# tu subsecond time (milliseconds)
+# tl Local time. Optional strftime format argument
+# default %d/%b/%Y:%H:%M:%S %z
+# tg GMT time. Optional strftime format argument
+# default %d/%b/%Y:%H:%M:%S %z
+# tr Response time (milliseconds)
+# >h Request header. Optional header name argument
+# on the format header[:[separator]element]
+# <h Reply header. Optional header name argument
+# as for >h
+# un User name
+# ul User name from authentication
+# ui User name from ident
+# us User name from SSL
+# ue User name from external acl helper
+# Hs HTTP status code
+# Ss Squid request status (TCP_MISS etc)
+# Sh Squid hierarchy status (DEFAULT_PARENT etc)
+# mt MIME content type
+# rm Request method (GET/POST etc)
+# ru Request URL
+# rv Request protocol version
+# ea Log string returned by external acl
+# <st Reply size including HTTP headers
+# >st Request size including HTTP headers
+# st Request+Reply size including HTTP headers
+# % a literal % character
+#
+# logformat squid %ts.%03tu %6tr %>a %Ss/%03Hs %<st %rm %ru %un %Sh/%<A %mt
+# logformat squidmime %ts.%03tu %6tr %>a %Ss/%03Hs %<st %rm %ru %un %Sh/%<A %mt [%>h] [%<h]
+# logformat common %>a %ui %un [%tl] "%rm %ru HTTP/%rv" %Hs %<st %Ss:%Sh
+# logformat combined %>a %ui %un [%tl] "%rm %ru HTTP/%rv" %Hs %<st "%{Referer}>h" "%{User-Agent}>h" %Ss:%Sh
+
+ def self.expressions
+ ret = []
+
+ # Jun 10 06:38:19 hill squid[2573]: 1213072699.830 0 192.168.61.210 TCP_NEGATIVE_HIT/404 628 GET http://www.ossim.net/download/debian/Release.gpg - NONE/- text/html
+
+ # squid combined combined_logintas
+ # seconds_since_epoch: %ts * (local_time)
+ # subsecond_time: %03tu *
+ # response_time_milliseconds: %6tr * *
+ # client_source_ip: %>a * *
+ # request_status: %Ss * *
+ # http_status_code: %03Hs * *
+ # reply_size: %<st * *
+ # request_method: %rm * *
+ # request_url: %ru * *
+ # user_name %un * *
+ # hierarchy_status: %Sh * *
+ # server_ip: %<A * *
+ # mime_type: %mt * *
+ #COMBINED NEW:
+ # user_indent: %ui *
+ # protocol_version: %rv *
+ # referer: %{Referer}>h *
+ # user_agent: %{User-Agent}>h *
+ #LOGINTAS COMBINED NEW:
+ # user_auth: %ul
+ # user_acl: %ue
+ # acl_log: %ea
+ # client_fqdn: %>A
+
+
+
+ # logformat squid %ts.%03tu %6tr %>a %Ss/%03Hs %<st %rm %ru %un %Sh/%<A %mt
+ ret.push({ :regex => /^(squid\[([^\]]+)\]:\s+)?(\d+)\.(\d\d\d)\s+(\d+) ([^ ]+) ([^\/]+)\/(\d+) (\d+) ([^ ]+) ([^ ]+) ([^ ]+) ([^\/]+)\/([^ ]+) (.+)\s*$/,
+ :fields => [nil,:process_id,:seconds_since_epoch, :subsecond_time, :response_time_milliseconds,
+ :client_source_ip, :request_status, :http_status_code, :reply_size,
+ :request_method, :request_url, :user_name, :hierarchy_status, :client_fqdn, :mime_type]
+ })
+
+ # one of syslogs default format
+ # squid combined format:
+ # logformat combined %>a %ui %un [%tl] "%rm %ru HTTP/%rv" %Hs %<st "%{Referer}>h" "%{User-Agent}>h" %Ss:%Sh
+ # 192.168.61.77 - - [29/Sep/2009:06:34:35 +0200] "GET http://debian.setup.in.here/mgmt-sarge/Release.gpg HTTP/1.0" 304 271 "-" "Debian APT-HTTP/1.3" TCP_REFRESH_HIT:DIRECT
+ ret.push({ :regex => /^(\d+\.\d+\.\d+\.\d+) ([^ ]+) ([^ ]+) \[([^\]]*)\] \"([^ ]+) ([^ ]+) HTTP\/([^\"]+)\" (\d+) (\d+) \"([^\"]*)\" \"([^\"]*)\" ([^:]+):([^ ]+) ?(.*)\n?$/,
+ :fields => [:client_source_ip, :user_indent, :user_name, :seconds_since_epoch,
+ :request_method, :request_url, :protocol_version, :http_status_code, :reply_size, :referer, :user_agent,
+ :request_status, :hierarchy_status, :message],
+ :result_filter => lambda {|results, meta_instance|
+ results[3] = Time.parse(DateTime.strptime(results[3],"%d/%b/%Y:%H:%M:%S %Z").to_s).to_i
+ results
+ }
+
+ })
+ end
+
+ def after_filling_values(values)
+ # get protocol and host from url
+ if %r{^(\w+)\://([^/]+)(/.*|$)$} =~ values[:request_url]
+ self.request_protocol = $1
+ self.request_host = $2
+ end
+
+ if self.message
+ raise "Get values of extended format"
+ # extendes logintas format
+ # logformat logintas_accesslog %>a %ui %un [%tl] "%rm %ru HTTP/%rv" %Hs %<st "%{Referer}>h" "%{User-Agent}>h" %Ss:%Sh [lit:] %ul %ue %ea request-header: %>A "%{Authorization}>h" "%{Cache-Control}>h" "%{From}>h" "%{Host}>h" "%{If-Modified-Since}>h" "%{If-Unmodified-Since}>h" "%{Pragma}>h" "%{Proxy-Authorization}>h" response-header: %<A %tr %mt "%{Server}<h" "%{Content-MD5}<h" "%{Age}<h" "%{Cache-Control}<h" "%{Content-Encoding}<h" "%{Content-Language}<h" "%{Date}<h" "%{Last-Modified}>h" "%{Location}>h" "%{Pragma}<h" "%{Proxy-Authenticate}<h" "%{Via}<h" "%{WWW-Authenticate}<h"
+ #
+ # extends the combined format with
+ # [lit:] %ul %ue %ea request-header: %>A "%{Authorization}>h" "%{Cache-Control}>h" "%{From}>h" "%{Host}>h" "%{If-Modified-Since}>h" "%{If-Unmodified-Since}>h" "%{Pragma}>h" "%{Proxy-Authorization}>h" response-header: %<A %tr %mt "%{Server}<h" "%{Content-MD5}<h" "%{Age}<h" "%{Cache-Control}<h" "%{Content-Encoding}<h" "%{Content-Language}<h" "%{Date}<h" "%{Last-Modified}>h" "%{Location}>h" "%{Pragma}<h" "%{Proxy-Authenticate}<h" "%{Via}<h" "%{WWW-Authenticate}<h"
+ #
+
+ end
+
+ end
+
+end
+
Added: incubator/alois/trunk/prisma/data/prisma/prisma_database/model/squid_request_header_meta.rb
URL: http://svn.apache.org/viewvc/incubator/alois/trunk/prisma/data/prisma/prisma_database/model/squid_request_header_meta.rb?rev=1031127&view=auto
==============================================================================
--- incubator/alois/trunk/prisma/data/prisma/prisma_database/model/squid_request_header_meta.rb (added)
+++ incubator/alois/trunk/prisma/data/prisma/prisma_database/model/squid_request_header_meta.rb Thu Nov 4 18:27:22 2010
@@ -0,0 +1,43 @@
+# Copyright 2010 The Apache Software Foundation.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+class SquidRequestHeaderMeta < ActiveRecord::Base
+ description "Extended request header of squid messages"
+ sources ["SquidMeta"]
+ def self.may_have_messages?; false; end
+
+ def self.expressions
+ ret = []
+
+ # tear out request part of
+ # request-header: %>A "%{Authorization}>h" "%{Cache-Control}>h" "%{From}>h" "%{Host}>h" "%{If-Modified-Since}>h" "%{If-Unmodified-Since}>h" "%{Pragma}>h" "%{Proxy-Authorization}>h" response-header: %<A %tr %mt "%{Server}<h" "%{Content-MD5}<h" "%{Age}<h" "%{Cache-Control}<h" "%{Content-Encoding}<h" "%{Content-Language}<h" "%{Date}<h" "%{Last-Modified}>h" "%{Location}>h" "%{Pragma}<h" "%{Proxy-Authenticate}<h" "%{Via}<h" "%{WWW-Authenticate}<h"
+
+ #--request-header
+ # (client_fqdn: %>A) already in squid meta
+ # authorization: %{Authorization}>h
+ # cache_control: %{Cache-Control}>h
+ # from: %{From}>h
+ # host %{Host}>h
+ # if_modified_since: %{If-Modified-Since}>h
+ # if_unmodified_since: %{If-Unmodified-Since}>h
+ # pragma: %{Pragma}>h
+ # proxy_authorization: %{Proxy-Authorization}>h
+
+ ret.push({ :regex => /request-header: [^ ]+ \"([^\"]*)\" \"([^\"]*)\" \"([^\"]*)\" \"([^\"]*)\" \"([^\"]*)\" \"([^\"]*)\" \"([^\"]*)\" \"([^\"]*)\" response-header:/,
+ :fields => [:authorization, :cache_control, :from, :host,
+ :if_modified_since, :if_unmodified_since, :pragma, :proxy_authorization]
+ })
+ ret
+ end
+end
Added: incubator/alois/trunk/prisma/data/prisma/prisma_database/model/squid_response_header_meta.rb
URL: http://svn.apache.org/viewvc/incubator/alois/trunk/prisma/data/prisma/prisma_database/model/squid_response_header_meta.rb?rev=1031127&view=auto
==============================================================================
--- incubator/alois/trunk/prisma/data/prisma/prisma_database/model/squid_response_header_meta.rb (added)
+++ incubator/alois/trunk/prisma/data/prisma/prisma_database/model/squid_response_header_meta.rb Thu Nov 4 18:27:22 2010
@@ -0,0 +1,52 @@
+# Copyright 2010 The Apache Software Foundation.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+class SquidResponseHeaderMeta < ActiveRecord::Base
+ description "Extended response header of squid messages"
+ sources ["SquidMeta"]
+ def self.may_have_messages?; false; end
+
+ def self.expressions
+ ret = []
+
+ # tear out response part of
+ # request-header: %>A "%{Authorization}>h" "%{Cache-Control}>h" "%{From}>h" "%{Host}>h" "%{If-Modified-Since}>h" "%{If-Unmodified-Since}>h" "%{Pragma}>h" "%{Proxy-Authorization}>h" response-header: %<A %tr %mt "%{Server}<h" "%{Content-MD5}<h" "%{Age}<h" "%{Cache-Control}<h" "%{Content-Encoding}<h" "%{Content-Language}<h" "%{Date}<h" "%{Last-Modified}>h" "%{Location}>h" "%{Pragma}<h" "%{Proxy-Authenticate}<h" "%{Via}<h" "%{WWW-Authenticate}<h"
+
+ #--response-header
+ # (server_ip: %<A) already in squid meta
+ # (response_time: %tr) already in squid meta
+ # (mime_type: %mt) already in squid meta
+ # server: %{Server}<h
+ # content_md5: %{Content-MD5}<h
+ # age: %{Age}<h
+ # cache_control: %{Cache-Control}<h
+ # content_encoding: %{Content-Encoding}<h
+ # content_language: %{Content-Language}<h
+ # date: %{Date}<h
+ # last_modified: %{Last-Modified}>h
+ # location: %{Location}>h
+ # pragma: %{Pragma}<h
+ # proxy_autheticate %{Proxy-Authenticate}<h
+ # via: %{Via}<h
+ # www_authenticate: %{WWW-Authenticate}<h
+
+
+ ret.push({ :regex => /response-header: [^ ]+ [^ ]+ [^ ]+ \"([^\"]*)\" \"([^\"]*)\" \"([^\"]*)\" \"([^\"]*)\" \"([^\"]*)\" \"([^\"]*)\" \"([^\"]*)\" \"([^\"]*)\" \"([^\"]*)\" \"([^\"]*)\" \"([^\"]*)\" \"([^\"]*)\" \"([^\"]*)\"$/,
+ :fields => [:server, :content_md5, :age, :cache_control, :content_encoding,
+ :content_language, :date, :last_modified, :location, :pragma, :proxy_authenticate, :via,
+ :www_authenticate]
+ })
+ ret
+ end
+end
Added: incubator/alois/trunk/prisma/data/prisma/prisma_database/model/syslogd_meta.rb
URL: http://svn.apache.org/viewvc/incubator/alois/trunk/prisma/data/prisma/prisma_database/model/syslogd_meta.rb?rev=1031127&view=auto
==============================================================================
--- incubator/alois/trunk/prisma/data/prisma/prisma_database/model/syslogd_meta.rb (added)
+++ incubator/alois/trunk/prisma/data/prisma/prisma_database/model/syslogd_meta.rb Thu Nov 4 18:27:22 2010
@@ -0,0 +1,90 @@
+# Copyright 2010 The Apache Software Foundation.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+ class SyslogdMeta < ActiveRecord::Base
+
+ description "Meta information for syslogd a syslogd log."
+ sources ["SourceDbMeta","ArchiveMeta"]
+
+ def original
+ # if not log_meta record is
+ # defined, we do not know the
+ # archive that the record is
+ # saved in, so exit
+ return [] unless log_meta
+
+ log_date = log_meta.date
+
+ if self.respond_to?(:queue_id) and (qid = self.queue_id)
+ # we know the id that should be in the archive
+ Archivator.find({:log_date => log_date,
+ :raw_class => SyslogdRaw,
+ :id=> qid})
+ else
+ # do the old way with regexps
+ regs = []
+ inv_classes = []
+ self.children_recursive.reverse.each {|ch|
+ inv_classes.push(ch.class) if ch.class.respond_to?(:invers_before_filter)
+
+ ch.class.columns.each {|column|
+ next if column.name =~ /^id$/ or column.name =~ /_id$/ or column.name =~ /meta_type_name/
+ val = ch.send(column.name)
+
+ inv_classes.each {|klass|
+ #p [val,klass.name]
+ val = klass.invers_before_filter(val)
+ #p val
+ }
+ val = val.strftime("%T") if column.name == "time" and val.class.name =~ /Time/
+ esc = Regexp.escape(val.to_s.gsub("\"","\\\\\\\\\\\""))
+ regs.push(Regexp.new(esc))
+ }
+ }
+ # regs.push(Regexp.new("time: \"#{log_meta.time}\"\n date: \"#{log_meta.date}\"\n"))
+ Archivator.find({:log_date => log_date,
+ :raw_class => SyslogdRaw,
+ :regexps => regs})
+ end
+ rescue
+ $!.message
+ end
+
+ def log_meta
+ LogMeta.find_by_syslogd_metas_id(self.id)
+ end
+
+ def SyslogdMeta.create_meta( source_meta, msg)
+ if msg.class == SyslogdRaw then
+ new_meta = SyslogdMeta.new.prisma_initialize(source_meta,
+ { :ip => msg.ip,
+ :facility => msg.facility,
+ :priority => msg.priority,
+ :level => msg.level,
+ :tag => msg.tag,
+ :program => msg.program,
+ :queue_id => msg.id
+ }
+ )
+
+ log_meta = LogMeta.new.prisma_initialize(new_meta, {
+ :date => msg.date,
+ :time => msg.time,
+ :host => msg.host,
+ :message => msg.msg} )
+ return log_meta
+ end
+ return nil
+ end
+ end
Added: incubator/alois/trunk/prisma/data/prisma/prisma_database/model/syslogd_small_base_meta.rb
URL: http://svn.apache.org/viewvc/incubator/alois/trunk/prisma/data/prisma/prisma_database/model/syslogd_small_base_meta.rb?rev=1031127&view=auto
==============================================================================
--- incubator/alois/trunk/prisma/data/prisma/prisma_database/model/syslogd_small_base_meta.rb (added)
+++ incubator/alois/trunk/prisma/data/prisma/prisma_database/model/syslogd_small_base_meta.rb Thu Nov 4 18:27:22 2010
@@ -0,0 +1,32 @@
+# Copyright 2010 The Apache Software Foundation.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+ class SyslogdSmallBaseMeta < ActiveRecord::Base
+
+ description "Small syslogd meta"
+ sources ["PureMeta", "LogMeta"]
+
+ def self.expressions
+ ret = []
+
+ ret.push({ :regex => /(.*?)\|(.*?)\|(.*?)\|(.*?)\|(.*)\|(.*)/,
+ :fields => [:date, :time, :level, :ip, :message,nil]})
+ end
+
+ def get_hash
+ return nil unless message
+ return "#{date} #{time} #{ip} #{message.msg}".hash
+ end
+
+ end
Added: incubator/alois/trunk/prisma/data/prisma/prisma_database/model/test_meta.rb
URL: http://svn.apache.org/viewvc/incubator/alois/trunk/prisma/data/prisma/prisma_database/model/test_meta.rb?rev=1031127&view=auto
==============================================================================
--- incubator/alois/trunk/prisma/data/prisma/prisma_database/model/test_meta.rb (added)
+++ incubator/alois/trunk/prisma/data/prisma/prisma_database/model/test_meta.rb Thu Nov 4 18:27:22 2010
@@ -0,0 +1,36 @@
+# Copyright 2010 The Apache Software Foundation.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+ class TestMeta < ActiveRecord::Base
+
+ description "Testing class"
+ sources ["PureMeta", "LogMeta"]
+
+ preseed_expression /^(PRISMA THROW EXCEPTION)$/
+
+ def initialize(parent, values)
+ super()
+ self.message = values[:test_msg]
+ self.save
+ throw "#{values[:test_msg]} This error is ok. Its for stability testing."
+ end
+
+ def self.expressions
+ ret = []
+
+ ret.push({ :regex => /^(PRISMA THROW EXCEPTION)$/,
+ :fields => [:test_msg]})
+ end
+
+ end
Added: incubator/alois/trunk/prisma/data/prisma/prisma_database/model/ulogd_nfct_meta.rb
URL: http://svn.apache.org/viewvc/incubator/alois/trunk/prisma/data/prisma/prisma_database/model/ulogd_nfct_meta.rb?rev=1031127&view=auto
==============================================================================
--- incubator/alois/trunk/prisma/data/prisma/prisma_database/model/ulogd_nfct_meta.rb (added)
+++ incubator/alois/trunk/prisma/data/prisma/prisma_database/model/ulogd_nfct_meta.rb Thu Nov 4 18:27:22 2010
@@ -0,0 +1,58 @@
+# Copyright 2010 The Apache Software Foundation.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+ class UlogdNfctMeta < ActiveRecord::Base
+
+ description "Ulogd Netfilter Connection Tracking Logs"
+ sources ["PureMeta", "LogMeta"]
+
+ preseed_expression /^ulogd\[/
+
+ def self.may_have_messages?; false; end
+
+ def self.expressions
+ ret = []
+
+ # ICMP:
+ # ulogd[9441]: [NEW] ORIG: SRC=192.168.130.1 DST=192.168.130.33 PROTO=ICMP TYPE=0 CODE=8
+ # PKTS=1 BYTES=84 , REPLY: SRC=192.168.130.33 DST=192.168.130.1
+ # PROTO=ICMP TYPE=0 CODE=8 PKTS=0 BYTES=0
+ # ulogd[9441]: [DESTROY] ORIG: SRC=192.168.130.1 DST=192.168.130.60 PROTO=ICMP TYPE=0 CODE=8
+ # PKTS=1 BYTES=84 , REPLY: SRC=192.168.130.60 DST=192.168.130.1
+ # PROTO=ICMP TYPE=0 CODE=8 PKTS=0 BYTES=0
+ ret.push({ :regex => /^ulogd\[([^\]]*)\]:\s+\[([^\]]+)\]\s+ORIG:\s+SRC=(\S+)\s+DST=(\S+)\s+PROTO=(ICMP)\s+TYPE=(\d+)\s+CODE=(\d+)\s+PKTS=(\d+)\s+BYTES=(\d+)\s+,\s+REPLY:\s+SRC=(\S+)\s+DST=(\S+)\s+PROTO=(ICMP)\s+TYPE=(\d+)\s+CODE=(\d+)\s+PKTS=(\d+)\s+BYTES=(\d+)/,
+ :fields => [:process_id, :event, :orig_saddr, :orig_daddr, :orig_protocol, :icmp_type, :icmp_code, :orig_pktcount, :orig_pktlen, :reply_saddr, :reply_daddr, :reply_protocol, nil, nil, :reply_pktcount, :reply_pktlen]})
+
+ # UDP:
+ # ulogd[9441]: [NEW] ORIG: SRC=192.168.61.182 DST=192.168.61.130 PROTO=UDP SPT=123 DPT=123
+ # PKTS=1 BYTES=76 , REPLY: SRC=192.168.61.130 DST=192.168.61.182
+ # PROTO=UDP SPT=123 DPT=123 PKTS=0 BYTES=0
+ # ulogd[9441]: [DESTROY] ORIG: SRC=192.168.130.60 DST=192.168.61.54 PROTO=UDP SPT=48875 DPT=53
+ # PKTS=8 BYTES=580 , REPLY: SRC=192.168.61.54 DST=192.168.130.60
+ # PROTO=UDP SPT=53 DPT=48875 PKTS=8 BYTES=1515
+ #
+ # TCP:
+ # ulogd[9441]: [NEW] ORIG: SRC=192.168.130.64 DST=192.168.61.58 PROTO=TCP SPT=4506 DPT=8080
+ # PKTS=1 BYTES=60 , REPLY: SRC=192.168.61.58 DST=192.168.130.64
+ # PROTO=TCP SPT=8080 DPT=4506 PKTS=0 BYTES=0
+ # ulogd[9441]: [DESTROY] ORIG: SRC=192.168.130.60 DST=192.168.61.58 PROTO=TCP SPT=37971 DPT=8080
+ # PKTS=5 BYTES=1377 , REPLY: SRC=192.168.61.58 DST=192.168.130.60
+ # PROTO=TCP SPT=8080 DPT=37971 PKTS=5 BYTES=966
+ ret.push({ :regex => /^ulogd\[([^\]]*)\]:\s+\[([^\]]+)\]\s+ORIG:\s+SRC=(\S+)\s+DST=(\S+)\s+PROTO=(\S+)\s+SPT=(\d+)\s+DPT=(\d+)\s+PKTS=(\d+)\s+BYTES=(\d+)\s+,\s+REPLY:\s+SRC=(\S+)\s+DST=(\S+)\s+PROTO=(\S+)\s+SPT=(\d+)\s+DPT=(\d+)\s+PKTS=(\d+)\s+BYTES=(\d+)/,
+ :fields => [:process_id, :event, :orig_saddr, :orig_daddr, :orig_protocol, :orig_sport, :orig_dport, :orig_pktcount, :orig_pktlen, :reply_saddr, :reply_daddr, :reply_protocol, :reply_sport, :reply_dport, :reply_pktcount, :reply_pktlen]})
+
+ return ret
+ end
+
+ end
Added: incubator/alois/trunk/prisma/data/prisma/prisma_database/model/windows_event_meta.rb
URL: http://svn.apache.org/viewvc/incubator/alois/trunk/prisma/data/prisma/prisma_database/model/windows_event_meta.rb?rev=1031127&view=auto
==============================================================================
--- incubator/alois/trunk/prisma/data/prisma/prisma_database/model/windows_event_meta.rb (added)
+++ incubator/alois/trunk/prisma/data/prisma/prisma_database/model/windows_event_meta.rb Thu Nov 4 18:27:22 2010
@@ -0,0 +1,98 @@
+# Copyright 2010 The Apache Software Foundation.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+ # references:
+ # http://support.microsoft.com/kb/308427: How to view and manage event logs in Event Viewer in Windows XP
+ # http://community.netscape.com/n/pfx/forum.aspx?tsn=3&nav=messages&webtag=ws-winprohelp&tid=191184&redirCnt=1: Msg # 191184.3
+
+ class WindowsEventMeta < ActiveRecord::Base
+
+ description "Meta base information for windows event logs."
+ sources ["PureMeta", "LogMeta"]
+
+ preseed_expression /^(Master MSWinEventLog|Replika MSWinEventLog)/
+
+ def WindowsEventMeta.convert_to_unicode(s)
+ while s =~ /377777777(\d\d)/
+ s.sub!("377777777#{$1}", eval("\"\\303\\2#{$1}\""))
+ end
+ return s
+ end
+
+ def WindowsEventMeta.convert_to_wincode(s)
+ ret = ""
+ two_byte = false
+ s.each_byte {|b|
+ if two_byte and b.to_s(8) =~ /2(\d\d)/ then
+ ret += "377777777#{$1}"
+ else
+ if b.to_s(8) == "303"
+ two_byte = true
+ else
+ ret += eval "\"\\#{b.to_s(8)}\""
+ end
+ end
+ }
+
+ return ret
+ end
+
+ def WindowsEventMeta.create_meta( source_meta, message)
+ $log.debug("WIN Got message: #{message}")
+ unless message.msg =~ self.preseed_expression
+ $log.warn("Cannot create windows meta out of: #{message.msg}")
+ return nil
+ end
+
+ splited = message.msg.split("\00011")
+ unless splited.length == 15
+ $log.info{"Windows message has not 15 parts separated by \\00011 trying \\t."}
+ splited = message.msg.split("\t")
+ end
+
+ unless splited.length == 15
+ $log.info{"Windows message has not 15 parts separated by \\00011 trying to separate by at least doublspace."}
+ splited = message.msg.split(" ").map {|m| m.strip}.reject {|m| m == ""}
+ end
+
+ unless splited.length == 15
+ $log.warn{"Cannot split message by \\00011 nor by tab nor by double spacing algorithm. giving up. (#{splited.inspect})"}
+ return nil
+ end
+
+ splited.each_with_index{|m,i| splited[i] = convert_to_unicode(m)}
+
+ datetime = DateTime.strptime(splited[4],"%a %b %d %T %Y")
+
+ WindowsEventMeta.new.prisma_initialize(source_meta,
+ { :log_name => splited[0], # ["Master MSWinEventLog", "Replika MSWinEventLog"]
+ :field1 => splited[1], # ["1"]
+ :event_type => splited[2], # ["Application", "System", "Security"]
+ :field3 => splited[3], # num
+ :date => datetime,:time => Time.parse(datetime.to_s), #4 date Wed Aug 08 10:07:16 2007
+ :event_id => splited[5], # ["1011", "15224", "15223", "452", "453", "40960", "146", "11166", "6013", "17101", "528", "680", "538", "15221", "7", "7035"]
+ :source => splited[6], # ["ACESERVER6.1", "SDSERV_PROGRESS", "LSASRV", "DnsApi", "EventLog", "ACECLIENT", "Security", "Norton AntiVirus", "Service Control Manager"]
+ :user => splited[7], # ["Unknown User", "SDesk", "SYSTEM"]
+ :category => splited[8], # ["N/A", "User"]
+ :level => splited[9], ## ["Information", "Warning", "Success Audit"]
+ :computer => splited[10], # ["MSP43", "MSP44"]
+ :facility => splited[11], #11 ["Devices", "Shell", "Printers", "None", "Disk", "Logon/Logoff", "Account Logon"]
+ :data => splited[12], #12 trace_binary
+ :field14=> splited[14], #14 num
+
+ :message => splited[13] #13 message
+ })
+ end
+ end
+
Added: incubator/alois/trunk/prisma/data/prisma/prisma_database/schema.rb
URL: http://svn.apache.org/viewvc/incubator/alois/trunk/prisma/data/prisma/prisma_database/schema.rb?rev=1031127&view=auto
==============================================================================
--- incubator/alois/trunk/prisma/data/prisma/prisma_database/schema.rb (added)
+++ incubator/alois/trunk/prisma/data/prisma/prisma_database/schema.rb Thu Nov 4 18:27:22 2010
@@ -0,0 +1,747 @@
+# Copyright 2010 The Apache Software Foundation.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# This file is auto-generated from the current state of the database. Instead of editing this file,
+# please use the migrations feature of Active Record to incrementally modify your database, and
+# then regenerate this schema definition.
+#
+# Note that this schema.rb definition is the authoritative source for your database schema. If you need
+# to create the application database on another system, you should be using db:schema:load, not running
+# all the migrations from scratch. The latter is a flawed and unsustainable approach (the more migrations
+# you'll amass, the slower it'll run and the greater likelihood for issues).
+#
+# It's strongly recommended to check this file into your version control system.
+
+ActiveRecord::Schema.define(:version => 0) do
+
+ create_table "ace_passcode_metas", :force => true do |t|
+ t.string "action", :limit => 40
+ t.string "login", :limit => 30
+ t.string "user_name", :limit => 100
+ t.string "token", :limit => 15
+ t.string "group_name", :limit => 100
+ t.string "site", :limit => 100
+ t.string "agent_host", :limit => 100
+ t.string "server", :limit => 100
+ t.integer "windows_event_metas_id"
+ t.integer "log_metas_id"
+ end
+
+ add_index "ace_passcode_metas", ["action"], :name => "ace_passcode_metas_action_index"
+ add_index "ace_passcode_metas", ["agent_host"], :name => "ace_passcode_metas_agent_host_index"
+ add_index "ace_passcode_metas", ["group_name"], :name => "ace_passcode_metas_group_index"
+ add_index "ace_passcode_metas", ["log_metas_id"], :name => "ace_passcode_metas_log_metas_id_index"
+ add_index "ace_passcode_metas", ["login"], :name => "ace_passcode_metas_login_index"
+ add_index "ace_passcode_metas", ["server"], :name => "ace_passcode_metas_server_index"
+ add_index "ace_passcode_metas", ["site"], :name => "ace_passcode_metas_site_index"
+ add_index "ace_passcode_metas", ["user_name"], :name => "ace_passcode_metas_user_name_index"
+ add_index "ace_passcode_metas", ["windows_event_metas_id"], :name => "ace_passcode_metas_windows_event_metas_id_index"
+
+ create_table "amavis_metas", :force => true do |t|
+ t.integer "process_id"
+ t.string "amavis_id", :limit => 20
+ t.string "action", :limit => 20
+ t.string "status", :limit => 20
+ t.string "from_field", :limit => 50
+ t.string "to_field"
+ t.string "message_id", :limit => 50
+ t.string "hits", :limit => 10
+ t.integer "process_time"
+ t.string "ip", :limit => 50
+ t.string "signature", :limit => 50
+ t.string "quarantine", :limit => 50
+ t.integer "pure_metas_id"
+ t.integer "log_metas_id"
+ end
+
+ add_index "amavis_metas", ["action"], :name => "amavis_metas_action_index"
+ add_index "amavis_metas", ["from_field"], :name => "amavis_metas_from_field_index"
+ add_index "amavis_metas", ["ip"], :name => "amavis_metas_ip_index"
+ add_index "amavis_metas", ["log_metas_id"], :name => "amavis_metas_log_metas_id_index"
+ add_index "amavis_metas", ["message_id"], :name => "amavis_metas_message_id_index"
+ add_index "amavis_metas", ["pure_metas_id"], :name => "amavis_metas_pure_metas_id_index"
+ add_index "amavis_metas", ["signature"], :name => "amavis_metas_signature_index"
+ add_index "amavis_metas", ["status"], :name => "amavis_metas_status_index"
+
+ create_table "apache_file_metas", :force => true do |t|
+ t.string "virtual_host", :limit => 100
+ t.integer "file_metas_id"
+ end
+
+ add_index "apache_file_metas", ["file_metas_id"], :name => "apache_file_metas_file_metas_id_index"
+
+ create_table "apache_log_metas", :force => true do |t|
+ t.string "forensic_id", :limit => 30
+ t.integer "serve_time"
+ t.string "host", :limit => 50
+ t.integer "log_metas_id"
+ end
+
+ add_index "apache_log_metas", ["log_metas_id"], :name => "index_apache_log_metas_on_log_metas_id"
+
+ create_table "apache_metas", :force => true do |t|
+ t.string "remote_host", :limit => 40
+ t.string "remote_logname", :limit => 20
+ t.string "remote_user", :limit => 20
+ t.time "time"
+ t.date "date"
+ t.string "first_line", :limit => 512
+ t.integer "status"
+ t.integer "bytes"
+ t.string "referer", :limit => 40
+ t.string "useragent", :limit => 40
+ t.integer "log_metas_id"
+ t.integer "pure_metas_id"
+ end
+
+ add_index "apache_metas", ["first_line"], :name => "apache_metas_first_line_index"
+ add_index "apache_metas", ["log_metas_id"], :name => "apache_metas_log_metas_id_index"
+ add_index "apache_metas", ["pure_metas_id"], :name => "apache_metas_pure_metas_id_index"
+ add_index "apache_metas", ["remote_host"], :name => "apache_metas_remote_host_index"
+ add_index "apache_metas", ["remote_user"], :name => "apache_metas_remote_user_index"
+ add_index "apache_metas", ["status"], :name => "apache_metas_status_index"
+ add_index "apache_metas", ["useragent"], :name => "apache_metas_useragent_index"
+
+ create_table "archive_metas", :force => true do |t|
+ t.string "filename"
+ t.integer "current"
+ t.integer "total"
+ t.integer "todo"
+ t.boolean "finished", :default => false
+ t.datetime "created_at"
+ t.datetime "updated_at"
+ end
+
+ create_table "cisco_base_metas", :force => true do |t|
+ t.string "ip", :limit => 40
+ t.string "message_type", :limit => 10
+ t.integer "severity"
+ t.integer "message_number"
+ t.integer "log_metas_id"
+ t.integer "syslogd_small_base_metas_id"
+ end
+
+ add_index "cisco_base_metas", ["ip"], :name => "cisco_base_metas_ip_index"
+ add_index "cisco_base_metas", ["log_metas_id"], :name => "cisco_base_metas_log_metas_id_index"
+ add_index "cisco_base_metas", ["message_type"], :name => "cisco_base_metas_message_type_index"
+ add_index "cisco_base_metas", ["severity"], :name => "cisco_base_metas_severity_index"
+ add_index "cisco_base_metas", ["syslogd_small_base_metas_id"], :name => "cisco_base_metas_syslogd_small_base_metas_id_index"
+
+ create_table "cisco_firewall_connection_metas", :force => true do |t|
+ t.string "msg", :limit => 30
+ t.string "reason", :limit => 30
+ t.integer "connection_id"
+ t.string "connection_type", :limit => 10
+ t.string "foreign_name", :limit => 30
+ t.string "foreign_ip", :limit => 40
+ t.string "foreign_port", :limit => 10
+ t.string "local_name", :limit => 30
+ t.string "local_ip", :limit => 40
+ t.string "local_port", :limit => 10
+ t.string "global_to_ip", :limit => 40
+ t.string "global_to_port", :limit => 10
+ t.string "global_from_ip", :limit => 40
+ t.string "global_from_port", :limit => 10
+ t.time "duration"
+ t.integer "bytes"
+ t.integer "cisco_base_metas_id"
+ t.integer "log_metas_id"
+ t.string "user", :limit => 20
+ end
+
+ add_index "cisco_firewall_connection_metas", ["cisco_base_metas_id"], :name => "cisco_firewall_connection_metas_cisco_base_metas_id_index"
+ add_index "cisco_firewall_connection_metas", ["connection_id"], :name => "cisco_firewall_connection_metas_connection_id_index"
+ add_index "cisco_firewall_connection_metas", ["connection_type"], :name => "cisco_firewall_connection_metas_connection_type_index"
+ add_index "cisco_firewall_connection_metas", ["foreign_ip"], :name => "cisco_firewall_connection_metas_foreign_ip_index"
+ add_index "cisco_firewall_connection_metas", ["foreign_name"], :name => "cisco_firewall_connection_metas_foreign_name_index"
+ add_index "cisco_firewall_connection_metas", ["foreign_port"], :name => "cisco_firewall_connection_metas_foreign_port_index"
+ add_index "cisco_firewall_connection_metas", ["global_from_ip"], :name => "cisco_firewall_connection_metas_global_from_ip_index"
+ add_index "cisco_firewall_connection_metas", ["global_from_port"], :name => "cisco_firewall_connection_metas_global_from_port_index"
+ add_index "cisco_firewall_connection_metas", ["global_to_ip"], :name => "cisco_firewall_connection_metas_global_to_ip_index"
+ add_index "cisco_firewall_connection_metas", ["global_to_port"], :name => "cisco_firewall_connection_metas_global_to_port_index"
+ add_index "cisco_firewall_connection_metas", ["local_ip"], :name => "cisco_firewall_connection_metas_local_ip_index"
+ add_index "cisco_firewall_connection_metas", ["local_name"], :name => "cisco_firewall_connection_metas_local_name_index"
+ add_index "cisco_firewall_connection_metas", ["local_port"], :name => "cisco_firewall_connection_metas_local_port_index"
+ add_index "cisco_firewall_connection_metas", ["log_metas_id"], :name => "cisco_firewall_connection_metas_log_metas_id_index"
+ add_index "cisco_firewall_connection_metas", ["reason"], :name => "cisco_firewall_connection_metas_reason_index"
+
+ create_table "cisco_firewall_metas", :force => true do |t|
+ t.string "msg", :limit => 100
+ t.string "source", :limit => 40
+ t.string "source_port", :limit => 10
+ t.string "destination", :limit => 40
+ t.string "destination_port", :limit => 10
+ t.string "interface", :limit => 20
+ t.integer "cisco_base_metas_id"
+ end
+
+ add_index "cisco_firewall_metas", ["cisco_base_metas_id"], :name => "cisco_firewall_metas_cisco_base_metas_id_index"
+ add_index "cisco_firewall_metas", ["destination"], :name => "cisco_firewall_metas_destination_index"
+ add_index "cisco_firewall_metas", ["destination_port"], :name => "cisco_firewall_metas_destination_port_index"
+ add_index "cisco_firewall_metas", ["interface"], :name => "cisco_firewall_metas_interface_index"
+ add_index "cisco_firewall_metas", ["source"], :name => "cisco_firewall_metas_source_index"
+ add_index "cisco_firewall_metas", ["source_port"], :name => "cisco_firewall_metas_source_port_index"
+
+ create_table "cisco_metas", :force => true do |t|
+ t.string "msg", :limit => 100
+ t.string "server", :limit => 40
+ t.string "server_port", :limit => 10
+ t.string "name", :limit => 40
+ t.string "ip", :limit => 40
+ t.string "port", :limit => 10
+ t.string "user", :limit => 20
+ t.string "group_name", :limit => 20
+ t.string "reason", :limit => 100
+ t.integer "cisco_base_metas_id"
+ end
+
+ add_index "cisco_metas", ["cisco_base_metas_id"], :name => "cisco_metas_cisco_base_metas_id_index"
+ add_index "cisco_metas", ["group_name"], :name => "cisco_metas_group_name_index"
+ add_index "cisco_metas", ["ip"], :name => "cisco_metas_ip_index"
+ add_index "cisco_metas", ["msg"], :name => "cisco_metas_msg_index"
+ add_index "cisco_metas", ["name"], :name => "cisco_metas_name_index"
+ add_index "cisco_metas", ["port"], :name => "cisco_metas_port_index"
+ add_index "cisco_metas", ["server"], :name => "cisco_metas_server_index"
+ add_index "cisco_metas", ["server_port"], :name => "cisco_metas_server_port_index"
+ add_index "cisco_metas", ["user"], :name => "cisco_metas_user_index"
+
+ create_table "cisco_session_metas", :force => true do |t|
+ t.string "msg", :limit => 100
+ t.string "session_type", :limit => 30
+ t.time "duration"
+ t.integer "in_bytes"
+ t.integer "out_bytes"
+ t.integer "cisco_metas_id"
+ end
+
+ add_index "cisco_session_metas", ["cisco_metas_id"], :name => "index_cisco_session_metas_on_cisco_metas_id"
+ add_index "cisco_session_metas", ["msg"], :name => "index_cisco_session_metas_on_msg"
+ add_index "cisco_session_metas", ["session_type"], :name => "index_cisco_session_metas_on_session_type"
+
+ create_table "compression_metas", :force => true do |t|
+ t.string "extname"
+ t.string "inflate_command"
+ t.integer "inflated_size"
+ t.integer "deflated_size"
+ t.integer "file_metas_id"
+ end
+
+ create_table "cron_metas", :force => true do |t|
+ t.integer "process_id"
+ t.string "user", :limit => 20
+ t.integer "uid"
+ t.string "program", :limit => 20
+ t.string "action", :limit => 20
+ t.string "command"
+ t.integer "pure_metas_id"
+ t.integer "log_metas_id"
+ end
+
+ add_index "cron_metas", ["action"], :name => "cron_metas_action_index"
+ add_index "cron_metas", ["log_metas_id"], :name => "cron_metas_log_metas_id_index"
+ add_index "cron_metas", ["program"], :name => "cron_metas_program_index"
+ add_index "cron_metas", ["pure_metas_id"], :name => "cron_metas_pure_metas_id_index"
+ add_index "cron_metas", ["uid"], :name => "cron_metas_uid_index"
+ add_index "cron_metas", ["user"], :name => "cron_metas_user_index"
+
+ create_table "fetchmail_metas", :force => true do |t|
+ t.integer "process_id"
+ t.string "program", :limit => 20
+ t.string "action", :limit => 200
+ t.integer "pure_metas_id"
+ t.integer "log_metas_id"
+ end
+
+ add_index "fetchmail_metas", ["action"], :name => "fetchmail_metas_action_index"
+ add_index "fetchmail_metas", ["log_metas_id"], :name => "fetchmail_metas_log_metas_id_index"
+ add_index "fetchmail_metas", ["program"], :name => "fetchmail_metas_program_index"
+ add_index "fetchmail_metas", ["pure_metas_id"], :name => "fetchmail_metas_pure_metas_id_index"
+
+ create_table "file_metas", :force => true do |t|
+ t.string "dirname"
+ t.string "basename"
+ t.string "ftype", :limit => 20
+ t.integer "size"
+ t.datetime "mtime"
+ t.datetime "atime"
+ t.datetime "ctime"
+ t.integer "umask"
+ t.integer "uid"
+ t.integer "gid"
+ t.datetime "created_at"
+ t.datetime "updated_at"
+ t.string "options"
+ t.integer "source_db_metas_id"
+ t.integer "archive_metas_id"
+ end
+
+ add_index "file_metas", ["archive_metas_id"], :name => "file_metas_archive_metas_id_index"
+ add_index "file_metas", ["source_db_metas_id"], :name => "file_metas_source_db_metas_id_index"
+
+ create_table "inet_header_metas", :force => true do |t|
+ t.date "date"
+ t.time "time"
+ t.integer "milliseconds"
+ t.string "severity", :limit => 10
+ t.string "system_id", :limit => 30
+ t.string "msg_id", :limit => 30
+ t.string "user_id", :limit => 30
+ t.string "eventtype", :limit => 30
+ t.integer "pure_metas_id"
+ t.integer "log_metas_id"
+ t.string "client_ip", :limit => 32
+ t.string "server_ip", :limit => 32
+ t.string "session_id", :limit => 32
+ t.integer "hit_number"
+ t.integer "num_object_hits"
+ t.string "text1", :limit => 1024
+ t.string "text2", :limit => 1024
+ end
+
+ add_index "inet_header_metas", ["client_ip"], :name => "index_inet_header_metas_on_client_ip"
+ add_index "inet_header_metas", ["date"], :name => "index_inet_header_metas_on_date"
+ add_index "inet_header_metas", ["eventtype"], :name => "index_inet_header_metas_on_eventtype"
+ add_index "inet_header_metas", ["hit_number"], :name => "index_inet_header_metas_on_hit_number"
+ add_index "inet_header_metas", ["log_metas_id"], :name => "index_inet_header_metas_on_log_metas_id"
+ add_index "inet_header_metas", ["msg_id"], :name => "index_inet_header_metas_on_msg_id"
+ add_index "inet_header_metas", ["num_object_hits"], :name => "index_inet_header_metas_on_num_object_hits"
+ add_index "inet_header_metas", ["pure_metas_id"], :name => "index_inet_header_metas_on_pure_metas_id"
+ add_index "inet_header_metas", ["server_ip"], :name => "index_inet_header_metas_on_server_ip"
+ add_index "inet_header_metas", ["session_id"], :name => "index_inet_header_metas_on_session_id"
+ add_index "inet_header_metas", ["severity"], :name => "index_inet_header_metas_on_severity"
+ add_index "inet_header_metas", ["system_id"], :name => "index_inet_header_metas_on_system_id"
+ add_index "inet_header_metas", ["user_id"], :name => "index_inet_header_metas_on_user_id"
+
+ create_table "inet_object_metas", :force => true do |t|
+ t.string "objecttype", :limit => 30
+ t.string "object_id", :limit => 512
+ t.string "version", :limit => 30
+ t.string "filename", :limit => 256
+ t.string "description", :limit => 256
+ t.string "object_hashes", :limit => 256
+ t.string "object_url", :limit => 512
+ t.integer "inet_header_metas_id"
+ t.integer "inet_object_metas_id"
+ t.string "object_location", :limit => 512
+ end
+
+ add_index "inet_object_metas", ["filename"], :name => "index_inet_object_metas_on_filename"
+ add_index "inet_object_metas", ["inet_header_metas_id"], :name => "index_inet_object_metas_on_inet_header_metas_id"
+ add_index "inet_object_metas", ["inet_object_metas_id"], :name => "index_inet_object_metas_on_inet_object_metas_id"
+ add_index "inet_object_metas", ["object_hashes"], :name => "index_inet_object_metas_on_object_hashes"
+ add_index "inet_object_metas", ["object_id"], :name => "index_inet_object_metas_on_object_id"
+ add_index "inet_object_metas", ["object_url"], :name => "index_inet_object_metas_on_object_url"
+ add_index "inet_object_metas", ["objecttype"], :name => "index_inet_object_metas_on_objecttype"
+ add_index "inet_object_metas", ["version"], :name => "index_inet_object_metas_on_version"
+
+ create_table "iptables_firewall_metas", :force => true do |t|
+ t.string "rule", :limit => 10
+ t.string "src", :limit => 20
+ t.string "spt", :limit => 10
+ t.string "dst", :limit => 20
+ t.string "dpt", :limit => 10
+ t.string "custom", :limit => 20
+ t.string "in", :limit => 10
+ t.string "out", :limit => 10
+ t.string "physin", :limit => 10
+ t.string "physout", :limit => 10
+ t.integer "len"
+ t.string "tos", :limit => 10
+ t.string "prec", :limit => 10
+ t.integer "ttl"
+ t.integer "identifier"
+ t.string "proto", :limit => 10
+ t.string "additional", :limit => 20
+ t.integer "pure_metas_id"
+ t.integer "log_metas_id"
+ end
+
+ add_index "iptables_firewall_metas", ["dpt"], :name => "iptables_firewall_metas_dpt_index"
+ add_index "iptables_firewall_metas", ["dst"], :name => "iptables_firewall_metas_dst_index"
+ add_index "iptables_firewall_metas", ["log_metas_id"], :name => "iptables_firewall_metas_log_metas_id_index"
+ add_index "iptables_firewall_metas", ["proto"], :name => "iptables_firewall_metas_proto_index"
+ add_index "iptables_firewall_metas", ["pure_metas_id"], :name => "iptables_firewall_metas_pure_metas_id_index"
+ add_index "iptables_firewall_metas", ["rule"], :name => "iptables_firewall_metas_rule_index"
+ add_index "iptables_firewall_metas", ["spt"], :name => "iptables_firewall_metas_spt_index"
+ add_index "iptables_firewall_metas", ["src"], :name => "iptables_firewall_metas_src_index"
+
+ create_table "log_metas", :force => true do |t|
+ t.date "date"
+ t.time "time"
+ t.string "host"
+ t.integer "hash_value"
+ t.integer "syslogd_metas_id"
+ t.integer "pure_metas_id"
+ t.integer "file_metas_id"
+ end
+
+ add_index "log_metas", ["date"], :name => "log_metas_date_index"
+ add_index "log_metas", ["file_metas_id"], :name => "log_metas_file_metas_id_index"
+ add_index "log_metas", ["hash_value"], :name => "log_metas_hash_value_index"
+ add_index "log_metas", ["host"], :name => "log_metas_host_index"
+ add_index "log_metas", ["pure_metas_id"], :name => "log_metas_pure_metas_id_index"
+ add_index "log_metas", ["syslogd_metas_id"], :name => "log_metas_syslogd_metas_id_index"
+ add_index "log_metas", ["time"], :name => "index_log_metas_on_time"
+
+ create_table "messages", :force => true do |t|
+ t.integer "meta_id"
+ t.binary "msg"
+ t.string "meta_type_name", :limit => 100
+ end
+
+ add_index "messages", ["meta_id"], :name => "messages_meta_id_index"
+ add_index "messages", ["meta_type_name"], :name => "messages_meta_type_name_index"
+
+ create_table "metas", :force => true do |t|
+ t.integer "parent_id"
+ t.string "meta_type_name", :limit => 100
+ t.integer "meta_id"
+ end
+
+ add_index "metas", ["meta_id"], :name => "metas_meta_id_index"
+ add_index "metas", ["meta_type_name"], :name => "metas_meta_type_name_index"
+ add_index "metas", ["parent_id"], :name => "metas_parent_id_index"
+
+ create_table "nagios_metas", :force => true do |t|
+ t.string "msg_type", :limit => 50
+ t.string "probed_by_host", :limit => 50
+ t.string "affected_host", :limit => 50
+ t.string "service", :limit => 20
+ t.string "status", :limit => 20
+ t.string "unknown_1", :limit => 20
+ t.integer "unknown_2"
+ t.string "output"
+ t.integer "pure_metas_id"
+ t.integer "log_metas_id"
+ end
+
+ add_index "nagios_metas", ["affected_host"], :name => "nagios_metas_affected_host_index"
+ add_index "nagios_metas", ["log_metas_id"], :name => "nagios_metas_log_metas_id_index"
+ add_index "nagios_metas", ["msg_type"], :name => "nagios_metas_msg_type_index"
+ add_index "nagios_metas", ["probed_by_host"], :name => "nagios_metas_probed_by_host_index"
+ add_index "nagios_metas", ["pure_metas_id"], :name => "nagios_metas_pure_metas_id_index"
+ add_index "nagios_metas", ["service"], :name => "nagios_metas_service_index"
+ add_index "nagios_metas", ["status"], :name => "nagios_metas_status_index"
+
+ create_table "nonyms", :force => true do |t|
+ t.string "real_name", :limit => 20
+ end
+
+ add_index "nonyms", ["real_name"], :name => "nonyms_real_name_index"
+
+ create_table "ovpn_base_metas", :force => true do |t|
+ t.string "vpn", :limit => 20
+ t.integer "process_id"
+ t.string "client_ip", :limit => 50
+ t.integer "client_port"
+ t.string "cert", :limit => 50
+ t.string "msg_type", :limit => 50
+ t.string "msg"
+ t.string "client", :limit => 20
+ t.integer "pure_metas_id"
+ t.integer "log_metas_id"
+ end
+
+ add_index "ovpn_base_metas", ["cert"], :name => "ovpn_base_metas_cert_index"
+ add_index "ovpn_base_metas", ["client_ip"], :name => "ovpn_base_metas_client_ip_index"
+ add_index "ovpn_base_metas", ["client_port"], :name => "ovpn_base_metas_client_port_index"
+ add_index "ovpn_base_metas", ["log_metas_id"], :name => "ovpn_base_metas_log_metas_id_index"
+ add_index "ovpn_base_metas", ["msg_type"], :name => "ovpn_base_metas_msg_type_index"
+ add_index "ovpn_base_metas", ["pure_metas_id"], :name => "ovpn_base_metas_pure_metas_id_index"
+ add_index "ovpn_base_metas", ["vpn"], :name => "ovpn_base_metas_vpn_index"
+
+ create_table "postfix_detail_metas", :force => true do |t|
+ t.string "orig_to", :limit => 50
+ t.string "relay_host", :limit => 50
+ t.string "relay_ip", :limit => 50
+ t.float "delay"
+ t.integer "size"
+ t.integer "nrcpt"
+ t.string "status", :limit => 20
+ t.string "result_text", :limit => 200
+ t.integer "postfix_metas_id"
+ t.integer "relay_port"
+ t.float "delay_before_qmgr"
+ t.float "delay_in_qmgr"
+ t.float "delay_conn_setup"
+ t.float "delay_transmission"
+ t.string "dsn", :limit => 10
+ t.string "result", :limit => 20
+ t.string "result_mail_id", :limit => 10
+ end
+
+ add_index "postfix_detail_metas", ["delay"], :name => "altered_postfix_detail_metas_delay_index"
+ add_index "postfix_detail_metas", ["postfix_metas_id"], :name => "altered_postfix_detail_metas_postfix_metas_id_index"
+ add_index "postfix_detail_metas", ["relay_host"], :name => "altered_postfix_detail_metas_relay_host_index"
+ add_index "postfix_detail_metas", ["relay_ip"], :name => "altered_postfix_detail_metas_relay_ip_index"
+ add_index "postfix_detail_metas", ["status"], :name => "altered_postfix_detail_metas_status_index"
+
+ create_table "postfix_metas", :force => true do |t|
+ t.string "program", :limit => 10
+ t.integer "process_id"
+ t.string "mail_message_id", :limit => 15
+ t.string "action", :limit => 40
+ t.string "host", :limit => 50
+ t.string "ip", :limit => 50
+ t.integer "pure_metas_id"
+ t.integer "log_metas_id"
+ t.string "email_local_part", :limit => 50
+ t.string "message_id", :limit => 50
+ t.string "email_domain", :limit => 50
+ end
+
+ add_index "postfix_metas", ["action"], :name => "altered_postfix_metas_action_index"
+ add_index "postfix_metas", ["email_domain"], :name => "index_postfix_metas_on_email_domain"
+ add_index "postfix_metas", ["email_local_part"], :name => "index_postfix_metas_on_email_local_part"
+ add_index "postfix_metas", ["host"], :name => "altered_postfix_metas_host_index"
+ add_index "postfix_metas", ["ip"], :name => "altered_postfix_metas_ip_index"
+ add_index "postfix_metas", ["log_metas_id"], :name => "altered_postfix_metas_log_metas_id_index"
+ add_index "postfix_metas", ["mail_message_id"], :name => "altered_postfix_metas_mail_message_id_index"
+ add_index "postfix_metas", ["message_id"], :name => "index_postfix_metas_on_message_id"
+ add_index "postfix_metas", ["program"], :name => "altered_postfix_metas_program_index"
+ add_index "postfix_metas", ["pure_metas_id"], :name => "altered_postfix_metas_pure_metas_id_index"
+
+ create_table "pure_metas", :force => true do |t|
+ t.integer "file_metas_id"
+ t.integer "compression_metas_id"
+ end
+
+ add_index "pure_metas", ["compression_metas_id"], :name => "index_pure_metas_on_compression_metas_id"
+ add_index "pure_metas", ["file_metas_id"], :name => "pure_metas_file_metas_id_index"
+
+ create_table "source_db_metas", :force => true do |t|
+ t.string "process_type", :limit => 10
+ t.integer "start"
+ t.integer "current"
+ t.integer "total"
+ t.integer "todo"
+ t.integer "count"
+ t.string "raw_class_name", :limit => 20
+ t.boolean "execute_once"
+ t.integer "waiting_time"
+ t.boolean "finished", :default => false
+ t.datetime "created_at"
+ t.datetime "updated_at"
+ end
+
+ create_table "squid_metas", :force => true do |t|
+ t.integer "process_id"
+ t.integer "seconds_since_epoch"
+ t.integer "subsecond_time"
+ t.integer "response_time_milliseconds"
+ t.string "client_source_ip", :limit => 40
+ t.string "request_status", :limit => 30
+ t.integer "http_status_code"
+ t.integer "reply_size"
+ t.string "request_method", :limit => 10
+ t.text "request_url"
+ t.string "user_name", :limit => 40
+ t.string "hierarchy_status", :limit => 30
+ t.string "server_ip", :limit => 50
+ t.string "mime_type", :limit => 60
+ t.string "request_protocol", :limit => 10
+ t.string "request_host", :limit => 50
+ t.integer "pure_metas_id"
+ t.integer "log_metas_id"
+ t.string "protocol_version", :limit => 5
+ t.string "client_fqdn", :limit => 50
+ t.text "referer"
+ t.string "user_indent", :limit => 40
+ t.string "user_agent", :limit => 40
+ t.string "user_auth", :limit => 40
+ t.string "user_acl", :limit => 40
+ t.text "acl_log"
+ end
+
+ add_index "squid_metas", ["client_fqdn"], :name => "new_client_fqdn_index"
+ add_index "squid_metas", ["client_source_ip"], :name => "index_squid_metas_on_client_source_ip"
+ add_index "squid_metas", ["hierarchy_status"], :name => "index_squid_metas_on_hierarchy_status"
+ add_index "squid_metas", ["http_status_code"], :name => "index_squid_metas_on_http_status_code"
+ add_index "squid_metas", ["log_metas_id"], :name => "index_squid_metas_on_log_metas_id"
+ add_index "squid_metas", ["mime_type"], :name => "index_squid_metas_on_mime_type"
+ add_index "squid_metas", ["process_id"], :name => "index_squid_metas_on_process_id"
+ add_index "squid_metas", ["protocol_version"], :name => "index_squid_metas_on_protocol_version"
+ add_index "squid_metas", ["pure_metas_id"], :name => "index_squid_metas_on_pure_metas_id"
+ add_index "squid_metas", ["reply_size"], :name => "index_squid_metas_on_reply_size"
+ add_index "squid_metas", ["request_host"], :name => "index_squid_metas_on_request_host"
+ add_index "squid_metas", ["request_method"], :name => "index_squid_metas_on_request_method"
+ add_index "squid_metas", ["request_protocol"], :name => "index_squid_metas_on_request_protocol"
+ add_index "squid_metas", ["request_status"], :name => "index_squid_metas_on_request_status"
+ add_index "squid_metas", ["response_time_milliseconds"], :name => "index_squid_metas_on_response_time_milliseconds"
+ add_index "squid_metas", ["server_ip"], :name => "index_squid_metas_on_client_fqdn"
+ add_index "squid_metas", ["user_acl"], :name => "index_squid_metas_on_user_acl"
+ add_index "squid_metas", ["user_agent"], :name => "index_squid_metas_on_user_agent"
+ add_index "squid_metas", ["user_auth"], :name => "index_squid_metas_on_user_auth"
+ add_index "squid_metas", ["user_indent"], :name => "index_squid_metas_on_user_indent"
+ add_index "squid_metas", ["user_name"], :name => "index_squid_metas_on_user_name"
+
+ create_table "squid_request_header_metas", :force => true do |t|
+ t.string "authorization", :limit => 50
+ t.string "cache_control", :limit => 50
+ t.string "from", :limit => 50
+ t.string "host", :limit => 50
+ t.string "if_modified_since", :limit => 50
+ t.string "if_unmodified_since", :limit => 50
+ t.string "pragma", :limit => 50
+ t.string "proxy_authorization", :limit => 50
+ t.integer "squid_metas_id"
+ end
+
+ add_index "squid_request_header_metas", ["authorization"], :name => "index_squid_request_header_metas_on_authorization"
+ add_index "squid_request_header_metas", ["from"], :name => "index_squid_request_header_metas_on_from"
+ add_index "squid_request_header_metas", ["host"], :name => "index_squid_request_header_metas_on_host"
+ add_index "squid_request_header_metas", ["squid_metas_id"], :name => "index_squid_request_header_metas_on_squid_metas_id"
+
+ create_table "squid_response_header_metas", :force => true do |t|
+ t.string "server", :limit => 50
+ t.string "content_md5", :limit => 50
+ t.string "age", :limit => 50
+ t.string "cache_control", :limit => 50
+ t.string "content_encoding", :limit => 50
+ t.string "content_language", :limit => 50
+ t.date "date"
+ t.datetime "last_modified"
+ t.string "location", :limit => 50
+ t.string "pragma", :limit => 50
+ t.string "proxy_authenticate", :limit => 50
+ t.string "via", :limit => 50
+ t.string "www_authenticate", :limit => 50
+ t.integer "squid_metas_id"
+ end
+
+ add_index "squid_response_header_metas", ["age"], :name => "index_squid_response_header_metas_on_age"
+ add_index "squid_response_header_metas", ["content_encoding"], :name => "index_squid_response_header_metas_on_content_encoding"
+ add_index "squid_response_header_metas", ["content_language"], :name => "index_squid_response_header_metas_on_content_language"
+ add_index "squid_response_header_metas", ["date"], :name => "index_squid_response_header_metas_on_date"
+ add_index "squid_response_header_metas", ["server"], :name => "index_squid_response_header_metas_on_server"
+ add_index "squid_response_header_metas", ["squid_metas_id"], :name => "index_squid_response_header_metas_on_squid_metas_id"
+
+ create_table "syslogd_metas", :force => true do |t|
+ t.string "ip", :limit => 40
+ t.string "facility", :limit => 10
+ t.string "priority", :limit => 10
+ t.string "level", :limit => 10
+ t.string "tag", :limit => 10
+ t.integer "program"
+ t.integer "source_db_metas_id"
+ t.integer "archive_metas_id"
+ t.integer "queue_id"
+ end
+
+ add_index "syslogd_metas", ["archive_metas_id"], :name => "syslogd_metas_archive_metas_id_index"
+ add_index "syslogd_metas", ["ip"], :name => "syslogd_metas_ip_index"
+ add_index "syslogd_metas", ["program"], :name => "syslogd_metas_program_index"
+ add_index "syslogd_metas", ["queue_id"], :name => "index_syslogd_metas_on_queue_id"
+ add_index "syslogd_metas", ["source_db_metas_id"], :name => "syslogd_metas_source_db_metas_id_index"
+
+ create_table "syslogd_small_base_metas", :force => true do |t|
+ t.date "date"
+ t.time "time"
+ t.string "level", :limit => 10
+ t.string "ip", :limit => 40
+ t.integer "hash_value"
+ t.integer "pure_metas_id"
+ t.integer "log_metas_id"
+ end
+
+ add_index "syslogd_small_base_metas", ["date"], :name => "syslogd_small_base_metas_date_index"
+ add_index "syslogd_small_base_metas", ["hash_value"], :name => "syslogd_small_base_metas_hash_value_index"
+ add_index "syslogd_small_base_metas", ["ip"], :name => "syslogd_small_base_metas_ip_index"
+ add_index "syslogd_small_base_metas", ["level"], :name => "syslogd_small_base_metas_level_index"
+ add_index "syslogd_small_base_metas", ["log_metas_id"], :name => "syslogd_small_base_metas_log_metas_id_index"
+ add_index "syslogd_small_base_metas", ["pure_metas_id"], :name => "syslogd_small_base_metas_pure_metas_id_index"
+
+ create_table "test_metas", :force => true do |t|
+ t.string "message"
+ t.integer "pure_metas_id"
+ t.integer "log_metas_id"
+ end
+
+ add_index "test_metas", ["log_metas_id"], :name => "test_metas_log_metas_id_index"
+ add_index "test_metas", ["pure_metas_id"], :name => "test_metas_pure_metas_id_index"
+
+ create_table "ulogd_nfct_metas", :force => true do |t|
+ t.integer "process_id"
+ t.string "event", :limit => 16
+ t.string "orig_saddr", :limit => 50
+ t.string "orig_daddr", :limit => 50
+ t.string "orig_protocol", :limit => 10
+ t.integer "orig_sport"
+ t.integer "orig_dport"
+ t.integer "orig_pktlen"
+ t.integer "orig_pktcount"
+ t.string "reply_saddr", :limit => 50
+ t.string "reply_daddr", :limit => 50
+ t.string "reply_protocol", :limit => 10
+ t.integer "reply_sport"
+ t.integer "reply_dport"
+ t.integer "reply_pktlen"
+ t.integer "reply_pktcount"
+ t.integer "icmp_code"
+ t.integer "icmp_type"
+ t.integer "log_metas_id"
+ t.integer "pure_metas_id"
+ end
+
+ add_index "ulogd_nfct_metas", ["event"], :name => "index_ulogd_nfct_metas_on_event"
+ add_index "ulogd_nfct_metas", ["icmp_code"], :name => "index_ulogd_nfct_metas_on_icmp_code"
+ add_index "ulogd_nfct_metas", ["icmp_type"], :name => "index_ulogd_nfct_metas_on_icmp_type"
+ add_index "ulogd_nfct_metas", ["log_metas_id"], :name => "index_ulogd_nfct_metas_on_log_metas_id"
+ add_index "ulogd_nfct_metas", ["orig_daddr"], :name => "index_ulogd_nfct_metas_on_orig_daddr"
+ add_index "ulogd_nfct_metas", ["orig_dport"], :name => "index_ulogd_nfct_metas_on_orig_dport"
+ add_index "ulogd_nfct_metas", ["orig_pktcount"], :name => "index_ulogd_nfct_metas_on_orig_pktcount"
+ add_index "ulogd_nfct_metas", ["orig_protocol"], :name => "index_ulogd_nfct_metas_on_orig_protocol"
+ add_index "ulogd_nfct_metas", ["orig_saddr"], :name => "index_ulogd_nfct_metas_on_orig_saddr"
+ add_index "ulogd_nfct_metas", ["orig_sport"], :name => "index_ulogd_nfct_metas_on_orig_sport"
+ add_index "ulogd_nfct_metas", ["reply_daddr"], :name => "index_ulogd_nfct_metas_on_reply_daddr"
+ add_index "ulogd_nfct_metas", ["reply_dport"], :name => "index_ulogd_nfct_metas_on_reply_dport"
+ add_index "ulogd_nfct_metas", ["reply_pktcount"], :name => "index_ulogd_nfct_metas_on_reply_pktcount"
+ add_index "ulogd_nfct_metas", ["reply_pktlen"], :name => "index_ulogd_nfct_metas_on_reply_pktlen"
+ add_index "ulogd_nfct_metas", ["reply_protocol"], :name => "index_ulogd_nfct_metas_on_reply_protocol"
+ add_index "ulogd_nfct_metas", ["reply_saddr"], :name => "index_ulogd_nfct_metas_on_reply_saddr"
+ add_index "ulogd_nfct_metas", ["reply_sport"], :name => "index_ulogd_nfct_metas_on_reply_sport"
+
+ create_table "windows_event_metas", :force => true do |t|
+ t.string "log_name", :limit => 30
+ t.integer "field1"
+ t.string "event_type", :limit => 20
+ t.integer "field3"
+ t.date "date"
+ t.time "time"
+ t.integer "event_id"
+ t.string "source", :limit => 30
+ t.string "user"
+ t.string "category", :limit => 30
+ t.string "level", :limit => 30
+ t.string "computer"
+ t.string "facility", :limit => 30
+ t.binary "data"
+ t.integer "field14"
+ t.integer "pure_metas_id"
+ t.integer "log_metas_id"
+ end
+
+ add_index "windows_event_metas", ["category"], :name => "windows_event_metas_category_index"
+ add_index "windows_event_metas", ["computer"], :name => "windows_event_metas_computer_index"
+ add_index "windows_event_metas", ["date"], :name => "windows_event_metas_date_index"
+ add_index "windows_event_metas", ["event_type"], :name => "windows_event_metas_event_type_index"
+ add_index "windows_event_metas", ["facility"], :name => "windows_event_metas_facility_index"
+ add_index "windows_event_metas", ["level"], :name => "windows_event_metas_message_level_index"
+ add_index "windows_event_metas", ["log_metas_id"], :name => "windows_event_metas_log_metasid_index"
+ add_index "windows_event_metas", ["log_name"], :name => "windows_event_metas_log_name_index"
+ add_index "windows_event_metas", ["pure_metas_id"], :name => "windows_event_metas_pure_metas_id_index"
+ add_index "windows_event_metas", ["source"], :name => "windows_event_metas_source_index"
+ add_index "windows_event_metas", ["user"], :name => "windows_event_metas_user_index"
+
+end
Added: incubator/alois/trunk/prisma/data/prisma/pumpy_database/migrate/001_initialize_database.rb
URL: http://svn.apache.org/viewvc/incubator/alois/trunk/prisma/data/prisma/pumpy_database/migrate/001_initialize_database.rb?rev=1031127&view=auto
==============================================================================
--- incubator/alois/trunk/prisma/data/prisma/pumpy_database/migrate/001_initialize_database.rb (added)
+++ incubator/alois/trunk/prisma/data/prisma/pumpy_database/migrate/001_initialize_database.rb Thu Nov 4 18:27:22 2010
@@ -0,0 +1,67 @@
+# Copyright 2010 The Apache Software Foundation.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+class InitializeDatabase < ActiveRecord::Migration
+ def self.up
+
+ create_table "file_raws" do |t|
+ t.column "dirname", :string
+ t.column "basename", :string
+ t.column "ftype", :string, :limit => 20
+ t.column "size", :integer
+ t.column "mtime", :datetime
+ t.column "atime", :datetime
+ t.column "ctime", :datetime
+ t.column "umask", :integer
+ t.column "uid", :integer
+ t.column "gid", :integer
+ t.column "created_at", :datetime
+ t.column "updated_at", :datetime
+ t.column "options", :string
+ t.column "msg", :binary
+ end
+
+ create_table "syslogd_raws" do |t|
+ t.column "ip", :string, :limit => 40
+ t.column "host", :string
+ t.column "facility", :string, :limit => 10
+ t.column "priority", :string, :limit => 10
+ t.column "level", :string, :limit => 10
+ t.column "tag", :string, :limit => 10
+ t.column "date", :date
+ t.column "time", :time
+ t.column "program", :integer, :limit => 15
+ t.column "created_at", :datetime
+ t.column "msg", :string
+ end
+
+ create_table "raws_states" do |t|
+ t.column "table_name", :string, :limit => 20
+ t.column "count_limit", :integer
+ t.column "count_time", :float
+ t.column "count_value", :integer
+ t.column "delete_time", :float
+ t.column "delete_value", :integer
+ t.column "created_at", :datetime
+ t.column "updated_at", :datetime
+ end
+
+ add_index "raws_states", ["table_name"], :name => "raws_states_table_name_index"
+ end
+
+ def self.down
+ raise IrreversibleMigration
+ end
+
+end