From 6faaa7282be10f4117b529d6643aaec675875f6b Mon Sep 17 00:00:00 2001 From: janw Date: Tue, 29 Apr 2008 12:24:28 +0000 Subject: [PATCH] Added incremental package list updates. References #427 git-svn-id: https://oss.gonicus.de/repositories/gosa/trunk@10722 594d385d-05f5-0310-b6e9-bd551577e9d8 --- gosa-si/gosa-si-server | 141 +++++++++++++++++++++++++++++++++++------ 1 file changed, 120 insertions(+), 21 deletions(-) diff --git a/gosa-si/gosa-si-server b/gosa-si/gosa-si-server index acc399fb8..5432c1a8e 100755 --- a/gosa-si/gosa-si-server +++ b/gosa-si/gosa-si-server @@ -2095,10 +2095,11 @@ sub create_sources_list { sub run_create_packages_list_db { - my ($session, $heap) = @_[SESSION, HEAP]; + my ($kernel, $session, $heap) = @_[KERNEL, SESSION, HEAP]; my $session_id = $session->ID; my $task = POE::Wheel::Run->new( + Priority => +20, Program => sub {&create_packages_list_db(undef, undef, $session_id)}, StdoutEvent => "session_run_result", StderrEvent => "session_run_debug", @@ -2181,14 +2182,116 @@ sub create_packages_list_db { close (CONFIG); find(\&cleanup_and_extract, keys( %repo_dirs )); + &main::strip_packages_list_statements(); unshift @packages_list_statements, "VACUUM"; - unshift @packages_list_statements, "DELETE FROM $packages_list_tn"; $packages_list_db->exec_statementlist(\@packages_list_statements); unlink($packages_list_under_construction); daemon_log("$session_id INFO: create_packages_list_db: finished", 5); return; } +# This function should do some intensive task to minimize the db-traffic +sub strip_packages_list_statements { + my @existing_entries= @{$packages_list_db->exec_statement("SELECT * FROM $main::packages_list_tn")}; + my @new_statement_list=(); + my $hash; + my $insert_hash; + my $update_hash; + my $delete_hash; + my $local_timestamp=get_time(); + + foreach my $existing_entry (@existing_entries) { + $hash->{@{$existing_entry}[0]}->{@{$existing_entry}[1]}->{@{$existing_entry}[2]}= $existing_entry; + } + + foreach my $statement (@packages_list_statements) { + if($statement =~ /^INSERT/i) { + # Assign the values from the insert statement + my ($distribution,$package,$version,$section,$description,$template,$timestamp) = ($1,$2,$3,$4,$5,$6,$7) if $statement =~ + /^INSERT\s+?INTO\s+?$main::packages_list_tn\s+?VALUES\s*?\('(.*?)',\s*?'(.*?)',\s*?'(.*?)',\s*?'(.*?)',\s*?'(.*?)',\s*?'(.*?)',\s*?'(.*?)'\s*?\)$/si; + if(exists($hash->{$distribution}->{$package}->{$version})) { + # If section or description has changed, update the DB + if( + (! (@{$hash->{$distribution}->{$package}->{$version}}[3] eq $section)) or + (! (@{$hash->{$distribution}->{$package}->{$version}}[4] eq $description)) + ) { + @{$update_hash->{$distribution}->{$package}->{$version}} = ($distribution,$package,$version,$section,$description,undef); + } + } else { + # Insert a non-existing entry to db + @{$insert_hash->{$distribution}->{$package}->{$version}} = ($distribution,$package,$version,$section,$description,$template); + } + } elsif ($statement =~ /^UPDATE/i) { + my ($template,$package,$version) = ($1,$2,$3) if $statement =~ + /^update\s+?$main::packages_list_tn\s+?set\s+?template\s*?=\s*?'(.*?)'\s+?where\s+?package\s*?=\s*?'(.*?)'\s+?and\s+?version\s*?=\s*?'(.*?)'\s*?;$/si; + foreach my $distribution (keys %{$hash}) { + if(exists($insert_hash->{$distribution}->{$package}->{$version})) { + # update the insertion hash to execute only one query per package (insert instead insert+update) + @{$insert_hash->{$distribution}->{$package}->{$version}}[5]= $template; + } elsif(exists($hash->{$distribution}->{$package}->{$version})) { + if( ! (@{$hash->{$distribution}->{$package}->{$version}}[5] eq $template)) { + my $section; + my $description; + if(defined(@{$update_hash->{$distribution}->{$package}->{$version}}[3]) and + length(@{$update_hash->{$distribution}->{$package}->{$version}}[3]) > 0 ) { + $section= @{$update_hash->{$distribution}->{$package}->{$version}}[3]; + } + if(defined(@{$update_hash->{$distribution}->{$package}->{$version}}[4])) { + $description= @{$update_hash->{$distribution}->{$package}->{$version}}[4]; + } + @{$update_hash->{$distribution}->{$package}->{$version}} = ($distribution,$package,$version,$section,$description,$template); + } + } + } + } + } + + # TODO: Check for orphaned entries + + # unroll the insert_hash + foreach my $distribution (keys %{$insert_hash}) { + foreach my $package (keys %{$insert_hash->{$distribution}}) { + foreach my $version (keys %{$insert_hash->{$distribution}->{$package}}) { + push @new_statement_list, "INSERT INTO $main::packages_list_tn VALUES ('$distribution','$package','$version'," + ."'@{$insert_hash->{$distribution}->{$package}->{$version}}[3]'," + ."'@{$insert_hash->{$distribution}->{$package}->{$version}}[4]'," + ."'@{$insert_hash->{$distribution}->{$package}->{$version}}[5]'," + ."'$local_timestamp')"; + } + } + } + + # unroll the update hash + foreach my $distribution (keys %{$update_hash}) { + foreach my $package (keys %{$update_hash->{$distribution}}) { + foreach my $version (keys %{$update_hash->{$distribution}->{$package}}) { + my $set = ""; + if(defined(@{$update_hash->{$distribution}->{$package}->{$version}}[3])) { + $set .= "section = '@{$update_hash->{$distribution}->{$package}->{$version}}[3]', "; + } + if(defined(@{$update_hash->{$distribution}->{$package}->{$version}}[4])) { + $set .= "description = '@{$update_hash->{$distribution}->{$package}->{$version}}[4]', "; + } + if(defined(@{$update_hash->{$distribution}->{$package}->{$version}}[5])) { + $set .= "template = '@{$update_hash->{$distribution}->{$package}->{$version}}[5]', "; + } + if(defined($set) and length($set) > 0) { + $set .= "timestamp = '$local_timestamp'"; + } else { + next; + } + push @new_statement_list, + "UPDATE $main::packages_list_tn SET $set WHERE" + ." distribution = '$distribution'" + ." AND package = '$package'" + ." AND version = '$version'"; + } + } + } + + @packages_list_statements = @new_statement_list; +} + sub parse_package_info { my ($baseurl, $dist, $section, $session_id)= @_; @@ -2275,7 +2378,7 @@ sub parse_package { # Trigger for description if ($line =~ /^Description:\s/){ - ($description)= ($line =~ /^Description: (.*)$/); + ($description)= &encode_base64(($line =~ /^Description: (.*)$/)); next; } @@ -2325,26 +2428,22 @@ sub cleanup_and_extract { mkpath($dir); system( "dpkg -e '$File::Find::name' '$dir/DEBIAN'" ); - if( -f "$dir/DEBIAN/templates" ) { - - daemon_log("DEBUG: Found debconf templates in '$package' - $newver", 5); - - my $tmpl= ""; - { - local $/=undef; - open FILE, "$dir/DEBIAN/templates"; - $tmpl = &encode_base64(); - close FILE; - } - rmtree("$dir/DEBIAN/templates"); + if( -f "$dir/DEBIAN/templates" ) { - $sql= "update $main::packages_list_tn set template = '$tmpl' where package = '$package' and version = '$newver';"; + daemon_log("DEBUG: Found debconf templates in '$package' - $newver", 5); - } else { - $sql= "update $main::packages_list_tn set template = '' where package = '$package' and version = '$newver';"; - } + my $tmpl= ""; + { + local $/=undef; + open FILE, "$dir/DEBIAN/templates"; + $tmpl = &encode_base64(); + close FILE; + } + rmtree("$dir/DEBIAN/templates"); - push @packages_list_statements, $sql; + $sql= "update $main::packages_list_tn set template = '$tmpl' where package = '$package' and version = '$newver';"; + push @packages_list_statements, $sql; + } } return; @@ -2429,7 +2528,7 @@ $fai_release_db = GOSA::DBsqlite->new($fai_release_file_name); $fai_release_db->create_table($fai_release_tn, \@fai_release_col_names); # connect to packages_list_db -unlink($packages_list_file_name); +#unlink($packages_list_file_name); unlink($packages_list_under_construction); $packages_list_db = GOSA::DBsqlite->new($packages_list_file_name); $packages_list_db->create_table($packages_list_tn, \@packages_list_col_names); -- 2.30.2