Open-ILS/src/extras/import/pg_loader.pl

   1 #!/usr/bin/perl
   2 use strict;
   3
   4 use OpenSRF::System;
   5 use OpenSRF::EX qw/:try/;
   6 use OpenSRF::Utils::SettingsClient;
   7 use OpenILS::Utils::Fieldmapper;
   8 use OpenSRF::Utils::JSON;
   9 use FileHandle;
  10
  11 use Time::HiRes qw/time/;
  12 use Getopt::Long;
  13
  14 my @files;
  15 my ($config, $output, @auto, @order, @wipe, $quiet) =
  16         ('/openils/conf/opensrf_core.xml');
  17 my $nocommit = 0;
  18
  19 GetOptions( 'config=s'      => \$config,
  20             'output=s'      => \$output,
  21             'wipe=s'        => \@wipe,
  22             'autoprimary=s' => \@auto,
  23             'order=s'       => \@order,
  24             'nocommit|n'    => \$nocommit,
  25             'quiet'         => \$quiet,
  26 );
  27
  28 my %lineset;
  29 my %fieldcache;
  30
  31 OpenSRF::System->bootstrap_client( config_file => $config );
  32 Fieldmapper->import(IDL => OpenSRF::Utils::SettingsClient->new->config_value("IDL"));
  33
  34 my $count = 0;
  35 my $starttime = time;
  36 while ( my $rec = <> ) {
  37         next unless ($rec);
  38
  39         my $row;
  40         try {
  41                 $row = OpenSRF::Utils::JSON->JSON2perl($rec);
  42         } catch Error with {
  43                 my $e = shift;
  44                 warn "\n\n !!! Error : $e \n\n at or around line $count\n";
  45         };
  46         next unless ($row);
  47
  48         my $class = $row->class_name;
  49         my $hint = $row->json_hint;
  50
  51         if (!$lineset{$hint}) {
  52                 $lineset{$hint} = [];
  53                 my @cols = $row->real_fields;
  54                 if (grep { $_ eq $hint} @auto) {
  55                         @cols = grep { $_ ne $class->Identity } @cols;
  56                 }
  57
  58                 $fieldcache{$hint} =
  59                         { table => $class->Table,
  60                           sequence => $class->Sequence,
  61                           pkey => $class->Identity,
  62                           fields => \@cols,
  63                         };
  64
  65         #XXX it burnnnsssessss
  66         $fieldcache{$hint}{table} =~ s/\.full_rec/.real_full_rec/o if ($hint eq 'mfr');
  67         }
  68
  69         push @{ $lineset{$hint} }, [map { $row->$_ } @{ $fieldcache{$hint}{fields} }];
  70
  71         if (!$quiet && !($count % 500)) {
  72                 print STDERR "\r$count\t". $count / (time - $starttime);
  73         }
  74
  75         $count++;
  76 }
  77
  78 print STDERR "\nWriting file ...\n" if (!$quiet);
  79
  80 $output = '&STDOUT' unless ($output);
  81 $output = FileHandle->new(">$output") if ($output);
  82
  83 binmode($output,'utf8');
  84
  85 $output->print("SET CLIENT_ENCODING TO 'UNICODE';\n\n");
  86 $output->print("BEGIN;\n\n");
  87
  88 my $after_commit = '';
  89 for my $h (@order) {
  90         # continue if there was no data for this table
  91         next unless ($fieldcache{$h});
  92
  93         my $fields = join(',', @{ $fieldcache{$h}{fields} });
  94         $output->print( "DELETE FROM $fieldcache{$h}{table};\n" ) if (grep {$_ eq $h } @wipe);
  95         # Speed up loading of bib records
  96         $output->print( "COPY $fieldcache{$h}{table} ($fields) FROM STDIN;\n" );
  97
  98         for my $line (@{ $lineset{$h} }) {
  99                 my @data;
 100                 my $x = 0;
 101                 for my $d (@$line) {
 102                         if (!defined($d)) {
 103                                 $d = '\N';
 104                         } else {
 105                                 $d =~ s/\f/\\f/gos;
 106                                 $d =~ s/\n/\\n/gos;
 107                                 $d =~ s/\r/\\r/gos;
 108                                 $d =~ s/\t/\\t/gos;
 109                                 $d =~ s/\\/\\\\/gos;
 110                         }
 111                         if ($h eq 'bre' and $fieldcache{$h}{fields}[$x] eq 'quality') {
 112                                 $d = int($d) if ($d ne '\N');
 113                         }
 114                         push @data, $d;
 115                         $x++;
 116                 }
 117                 $output->print( join("\t", @data)."\n" );
 118         }
 119
 120         $output->print('\.'."\n\n");
 121
 122         if ($h eq 'mfr') {
 123                 $output->print("SELECT reporter.enable_materialized_simple_record_trigger();\n");
 124                 $output->print("SELECT reporter.disable_materialized_simple_record_trigger();\n");
 125         }
 126
 127         $after_commit .= "SELECT setval('$fieldcache{$h}{sequence}'::TEXT, (SELECT MAX($fieldcache{$h}{pkey}) FROM $fieldcache{$h}{table}), TRUE);\n"
 128                 if (!grep { $_ eq $h} @auto);
 129 }
 130
 131 $output->print("COMMIT;\n\n") unless $nocommit;
 132 $output->print($after_commit);
 133 $output->close;