Don't bother on empty statements in exec_statementlist.

Fix for event recreate_packages_list_db.
Speedup for creating packages lists.
Statements for creating packages lists are now executed as list.

git-svn-id: https://oss.gonicus.de/repositories/gosa/trunk@10619 594d385d-05f5-0310-b6e9-bd551577e9d8

diff --git a/gosa-si/gosa-si-server b/gosa-si/gosa-si-server
index 58bb3dbc26f7a986070e16bab72b95bf02529019..b2dff82b4bb49d06a5e8171260ef6cd6aef5e527 100755 (executable)
--- a/gosa-si/gosa-si-server
+++ b/gosa-si/gosa-si-server
@@ -107,7 +107,8 @@ $ping_timeout = 5;
 $no_bus = 0;
 $bus_activ = "true";
 $no_arp = 0;
-my $packages_list_under_construction = 0;
+my $packages_list_under_construction = "/tmp/packages_list_creation_in_progress";
+my @packages_list_statements;
 my $watch_for_new_jobs_in_progress = 0;
 
 our $prg= basename($0);
@@ -1248,10 +1249,11 @@ sub trigger_db_loop {
        my ($kernel) = @_ ;
        $kernel->delay_set('watch_for_new_jobs', $job_queue_loop_delay);
        $kernel->delay_set('watch_for_done_jobs', $job_queue_loop_delay); 
-    $kernel->delay_set('watch_for_new_messages', $messaging_db_loop_delay);
-    $kernel->delay_set('watch_for_done_messages', $messaging_db_loop_delay);
+       $kernel->delay_set('watch_for_new_messages', $messaging_db_loop_delay);
+       $kernel->delay_set('watch_for_done_messages', $messaging_db_loop_delay);
 }
 
+
 sub watch_for_done_jobs {
     my ($kernel,$heap) = @_[KERNEL, HEAP];
 
@@ -1268,6 +1270,7 @@ sub watch_for_done_jobs {
     $kernel->delay_set('watch_for_done_jobs',$job_queue_loop_delay);
 }
 
+
 sub watch_for_new_jobs {
        if($watch_for_new_jobs_in_progress == 0) {
                $watch_for_new_jobs_in_progress = 1;
@@ -1993,30 +1996,31 @@ sub session_run_done {
     delete $heap->{task}->{$task_id};
 }
 
+
 sub create_sources_list {
-    my ($session_id) = @_;
-    my $ldap_handle = &get_ldap_handle;
+       my $session_id = shift;
+       my $ldap_handle = &main::get_ldap_handle;
        my $result="/tmp/gosa_si_tmp_sources_list";
 
        # Remove old file
        if(stat($result)) {
                unlink($result);
-        &main::daemon_log("$session_id DEBUG: remove an old version of '$result'", 7); 
+               &main::daemon_log("$session_id DEBUG: remove an old version of '$result'", 7); 
        }
 
        my $fh;
        open($fh, ">$result");
-    if (not defined $fh) {
-        &main::daemon_log("$session_id DEBUG: cannot open '$result' for writing", 7); 
-        return undef;
-    }
-       if(defined($ldap_server_dn) and length($ldap_server_dn) > 0) {
+       if (not defined $fh) {
+               &main::daemon_log("$session_id DEBUG: cannot open '$result' for writing", 7); 
+               return undef;
+       }
+       if(defined($main::ldap_server_dn) and length($main::ldap_server_dn) > 0) {
                my $mesg=$ldap_handle->search(
-                               base    => $ldap_server_dn,
-                               scope   => 'base',
-                               attrs   => 'FAIrepository',
-                               filter  => 'objectClass=FAIrepositoryServer'
-                               );
+                       base    => $main::ldap_server_dn,
+                       scope   => 'base',
+                       attrs   => 'FAIrepository',
+                       filter  => 'objectClass=FAIrepositoryServer'
+               );
                if($mesg->count) {
                        foreach my $entry(@{$mesg->{'entries'}}) {
                                foreach my $value(@{$entry->get_value('FAIrepository', asref => 1)}) {
@@ -2029,8 +2033,8 @@ sub create_sources_list {
                        }
                }
        } else {
-        &main::daemon_log("$session_id ERROR: something wrong with ldap_server_dn '$ldap_server_dn', abort create_sources_list", 1); 
-    }
+               &main::daemon_log("$session_id ERROR: something wrong with ldap_server_dn '$main::ldap_server_dn', abort create_sources_list", 1); 
+       }
        close($fh);
 
        return $result;
@@ -2052,7 +2056,20 @@ sub run_create_packages_list_db {
 
 
 sub create_packages_list_db {
-    my ($ldap_handle, $sources_file, $session_id) = @_;
+       my ($ldap_handle, $sources_file, $session_id) = @_;
+       
+       # it should not be possible to trigger a recreation of packages_list_db
+       # while packages_list_db is under construction, so set flag packages_list_under_construction
+       # which is tested befor recreation can be started
+       if (-r $packages_list_under_construction) {
+               daemon_log("$session_id WARNING: packages_list_db is right now under construction, please wait until this process is finished", 3);
+               return;
+       } else {
+               daemon_log("$session_id INFO: create_packages_list_db: start", 5); 
+               # set packages_list_under_construction to true
+               system("touch $packages_list_under_construction");
+               @packages_list_statements=undef;
+       }
 
        if (not defined $session_id) { $session_id = 0; }
        if (not defined $ldap_handle) { 
@@ -2060,69 +2077,61 @@ sub create_packages_list_db {
 
                if (not defined $ldap_handle) {
                        daemon_log("$session_id ERROR: no ldap_handle available to create_packages_list_db", 1);
+                       unlink($packages_list_under_construction);
                        return;
                }
        }
-    if (not defined $sources_file) { 
-        &main::daemon_log("$session_id INFO: no sources_file given for creating packages list so trigger creation of it", 5); 
-        $sources_file = create_sources_list($session_id);
-    }
-
-    if (not defined $sources_file) {
-        &main::daemon_log("$session_id ERROR: no sources_file given under '$sources_file', skip create_packages_list_db", 1); 
-
-    }
-
-       # it should not be possible to trigger a recreation of packages_list_db
-       # while packages_list_db is under construction, so set flag packages_list_under_construction
-       # which is tested befor recreation can be started
-       if ($packages_list_under_construction) {
-                       daemon_log("$session_id WARNING: packages_list_db is right now under construction, please wait untill this process is finished", 3);
-                       return;
-       } else {
-                       daemon_log("$session_id INFO: create_packages_list_db: start", 5); 
-                       # set packages_list_under_construction to true
-                       $packages_list_under_construction = 1;
+       if (not defined $sources_file) { 
+               &main::daemon_log("$session_id INFO: no sources_file given for creating packages list so trigger creation of it", 5); 
+               $sources_file = &create_sources_list($session_id);
        }
-       my $line;
 
-    open(CONFIG, "<$sources_file") or do {
-        daemon_log( "$session_id ERROR: create_packages_list_db: Failed to open '$sources_file'", 1);
-        return;
-    };
-    
-    # Read lines
-    while ($line = <CONFIG>){
-        # Unify
-        chop($line);
-        $line =~ s/^\s+//;
-        $line =~ s/^\s+/ /;
-
-        # Strip comments
-        $line =~ s/#.*$//g;
+       if (not defined $sources_file) {
+               &main::daemon_log("$session_id ERROR: no sources_file given under '$sources_file', skip create_packages_list_db", 1); 
+               unlink($packages_list_under_construction);
+               return;
+       }
 
-        # Skip empty lines
-        if ($line =~ /^\s*$/){
-            next;
-        }
+       my $line;
 
-        # Interpret deb line
-        if ($line =~ /^deb [^\s]+\s[^\s]+\s[^\s]+/){
-            my( $baseurl, $dist, $sections ) = ($line =~ /^deb\s([^\s]+)\s+([^\s]+)\s+(.*)$/);
-            my $section;
-            foreach $section (split(' ', $sections)){
-                &parse_package_info( $baseurl, $dist, $section, $session_id );
-            }
-        }
-    }
+       open(CONFIG, "<$sources_file") or do {
+               daemon_log( "$session_id ERROR: create_packages_list_db: Failed to open '$sources_file'", 1);
+               unlink($packages_list_under_construction);
+               return;
+       };
+
+       # Read lines
+       while ($line = <CONFIG>){
+               # Unify
+               chop($line);
+               $line =~ s/^\s+//;
+               $line =~ s/^\s+/ /;
+
+               # Strip comments
+               $line =~ s/#.*$//g;
+
+               # Skip empty lines
+               if ($line =~ /^\s*$/){
+                       next;
+               }
 
-    close (CONFIG);
+               # Interpret deb line
+               if ($line =~ /^deb [^\s]+\s[^\s]+\s[^\s]+/){
+                       my( $baseurl, $dist, $sections ) = ($line =~ /^deb\s([^\s]+)\s+([^\s]+)\s+(.*)$/);
+                       my $section;
+                       foreach $section (split(' ', $sections)){
+                               &parse_package_info( $baseurl, $dist, $section, $session_id );
+                       }
+               }
+       }
 
-    daemon_log("$session_id INFO: create_packages_list_db: finished", 5); 
-       # set packages_list_under_construction to false
-       $packages_list_under_construction = 0;
+       close (CONFIG);
 
-    return;
+       find(\&cleanup_and_extract, keys( %repo_dirs ));
+       $packages_list_db->exec_statementlist(\@packages_list_statements);
+       unlink($packages_list_under_construction);
+       daemon_log("$session_id INFO: create_packages_list_db: finished", 5); 
+       return;
 }
 
 
@@ -2139,7 +2148,6 @@ sub parse_package_info {
         parse_package( "$outdir/$dist/$section", $dist, $path, $session_id );
     }
     
-    find(\&cleanup_and_extract, keys( %repo_dirs ));
 }
 
 
@@ -2167,7 +2175,6 @@ sub parse_package {
     my ($path, $dist, $srv_path, $session_id)= @_;
     if (not defined $session_id) { $session_id = 0;}
     my ($package, $version, $section, $description);
-    my @sql_list;
     my $PACKAGES;
     my $timestamp = &get_time();
 
@@ -2191,7 +2198,7 @@ sub parse_package {
         # Use empty lines as a trigger
         if ($line =~ /^\s*$/){
             my $sql = "INSERT INTO packages_list VALUES ('$dist', '$package', '$version', '$section', '', 'none', '$timestamp')";
-            push(@sql_list, $sql);
+            push(@packages_list_statements, $sql);
             $package = "none";
             $version = "none";
             $section = "none";
@@ -2234,9 +2241,6 @@ sub parse_package {
     close( $PACKAGES );
     unlink( "$path.in" );
     &main::daemon_log("$session_id DEBUG: unlink '$path.in'", 1); 
-    my $len_sql_list = @sql_list;
-    &main::daemon_log("$session_id DEBUG: add $len_sql_list insert-statements to packages_list_db", 5); 
-    $packages_list_db->exec_statementlist(\@sql_list);
 }
 
 
@@ -2285,7 +2289,7 @@ sub cleanup_and_extract {
             $sql= "update $main::packages_list_tn set template = '' where package = '$package' and version = '$newver';";
         }
 
-        my $res= $main::packages_list_db->update_dbentry($sql);
+        push @packages_list_statements, $sql;
     }
 
     return;
@@ -2371,6 +2375,7 @@ $fai_release_db->create_table($fai_release_tn, \@fai_release_col_names);
 
 # connect to packages_list_db
 unlink($packages_list_file_name);
+unlink($packages_list_under_construction);
 $packages_list_db = GOSA::DBsqlite->new($packages_list_file_name);
 $packages_list_db->create_table($packages_list_tn, \@packages_list_col_names);
 

diff --git a/gosa-si/modules/DBsqlite.pm b/gosa-si/modules/DBsqlite.pm
index 03b8d454baae95040d8ba7d842100e3ae86385cd..556b31ef7e96878077fe6793d4c0a0a73973aa3b 100644 (file)
--- a/gosa-si/modules/DBsqlite.pm
+++ b/gosa-si/modules/DBsqlite.pm
@@ -296,30 +296,35 @@ sub exec_statement {
 
 
 sub exec_statementlist {
-    my $self = shift;
-    my $sql_list = shift;
-    my @db_answer;
+       my $self = shift;
+       my $sql_list = shift;
+       my @db_answer;
 
-    foreach my $sql (@$sql_list) {
-               eval {
-               my @answer = @{$self->{dbh}->selectall_arrayref($sql)};
-                       push @db_answer, @answer;
-               };
-               if($@) {
-                       $self->{dbh}->do("ANALYZE");
+       foreach my $sql (@$sql_list) {
+               if(defined($sql) && length($sql) > 0) {
                        eval {
-                       my @answer = @{$self->{dbh}->selectall_arrayref($sql)};
+                               my @answer = @{$self->{dbh}->selectall_arrayref($sql)};
                                push @db_answer, @answer;
                        };
                        if($@) {
-                               &main::daemon_log("ERROR: $sql failed with $@", 1);
+                               $self->{dbh}->do("ANALYZE");
+                               eval {
+                                       my @answer = @{$self->{dbh}->selectall_arrayref($sql)};
+                                       push @db_answer, @answer;
+                               };
+                               if($@) {
+                                       &main::daemon_log("ERROR: $sql failed with $@", 1);
+                               }
                        }
+               } else {
+                       next;
                }
-    }
+       }
 
-    return \@db_answer;
+       return \@db_answer;
 }
 
+
 sub count_dbentries {
     my ($self, $table)= @_;
     my $error= 0;

diff --git a/gosa-si/server/events/gosaTriggered.pm b/gosa-si/server/events/gosaTriggered.pm
index 85006750acbcc88948a60fbeb3648a8d0a062662..f6950552ca4db9fc85df921cc43aafaf5f327883 100644 (file)
--- a/gosa-si/server/events/gosaTriggered.pm
+++ b/gosa-si/server/events/gosaTriggered.pm
@@ -247,6 +247,9 @@ sub recreate_fai_release_db {
 sub recreate_packages_list_db {
        my ($msg, $msg_hash, $session_id) = @_ ;
        my $out_msg;
+       my $header = @{$msg_hash->{'header'}}[0];
+       my $source = @{$msg_hash->{'source'}}[0];
+       my $target = @{$msg_hash->{'target'}}[0];
 
        my $jobdb_id = @{$msg_hash->{'jobdb_id'}}[0];
        if( defined $jobdb_id) {
@@ -255,7 +258,7 @@ sub recreate_packages_list_db {
                my $res = $main::job_db->exec_statement($sql_statement);
        }
 
-       &main::create_packages_list_db;
+       &main::create_packages_list_db(undef,undef,undef);
 
        my @out_msg_l = ( $out_msg );
        return @out_msg_l;