Open-ILS/src/extras/import/pg_loader.pl

   1 #!/usr/bin/perl
   2 use strict;
   3
   4 use lib '/openils/lib/perl5/';
   5
   6 use OpenSRF::System;
   7 use OpenSRF::EX qw/:try/;
   8 use OpenSRF::Utils::SettingsClient;
   9 use OpenILS::Utils::Fieldmapper;
  10 use OpenSRF::Utils::JSON;
  11 use FileHandle;
  12
  13 use Time::HiRes qw/time/;
  14 use Getopt::Long;
  15
  16 my @files;
  17 my ($config, $output, @auto, @order, @wipe, $quiet) =
  18         ('/openils/conf/opensrf_core.xml');
  19
  20 GetOptions(
  21         'config=s'      => \$config,
  22         'output=s'      => \$output,
  23         'wipe=s'        => \@wipe,
  24         'autoprimary=s' => \@auto,
  25         'order=s'       => \@order,
  26         'quiet'         => \$quiet,
  27 );
  28
  29 my %lineset;
  30 my %fieldcache;
  31
  32 OpenSRF::System->bootstrap_client( config_file => $config );
  33 Fieldmapper->import(IDL => OpenSRF::Utils::SettingsClient->new->config_value("IDL"));
  34
  35 my $count = 0;
  36 my $starttime = time;
  37 while ( my $rec = <> ) {
  38         next unless ($rec);
  39
  40         my $row;
  41         try {
  42                 $row = OpenSRF::Utils::JSON->JSON2perl($rec);
  43         } catch Error with {
  44                 my $e = shift;
  45                 warn "\n\n !!! Error : $e \n\n at or around line $count\n";
  46         };
  47         next unless ($row);
  48
  49         my $class = $row->class_name;
  50         my $hint = $row->json_hint;
  51
  52         if (!$lineset{$hint}) {
  53                 $lineset{$hint} = [];
  54                 my @cols = $row->real_fields;
  55                 if (grep { $_ eq $hint} @auto) {
  56                         @cols = grep { $_ ne $class->Identity } @cols;
  57                 }
  58
  59                 $fieldcache{$hint} =
  60                         { table => $class->Table,
  61                           sequence => $class->Sequence,
  62                           pkey => $class->Identity,
  63                           fields => \@cols,
  64                         };
  65         }
  66
  67         push @{ $lineset{$hint} }, [map { $row->$_ } @{ $fieldcache{$hint}{fields} }];
  68
  69         if (!$quiet && !($count % 500)) {
  70                 print STDERR "\r$count\t". $count / (time - $starttime);
  71         }
  72
  73         $count++;
  74 }
  75
  76 print STDERR "\nWriting file ...\n" if (!$quiet);
  77
  78 $output = '&STDOUT' unless ($output);
  79 $output = FileHandle->new(">$output") if ($output);
  80
  81 binmode($output,'utf8');
  82
  83 $output->print("SET CLIENT_ENCODING TO 'UNICODE';\n\n");
  84 $output->print("BEGIN;\n\n");
  85
  86 for my $h (@order) {
  87         # continue if there was no data for this table
  88         next unless ($fieldcache{$h});
  89
  90         my $fields = join(',', @{ $fieldcache{$h}{fields} });
  91         $output->print( "DELETE FROM $fieldcache{$h}{table};\n" ) if (grep {$_ eq $h } @wipe);
  92         $output->print( "COPY $fieldcache{$h}{table} ($fields) FROM STDIN;\n" );
  93
  94         for my $line (@{ $lineset{$h} }) {
  95                 my @data;
  96                 my $x = 0;
  97                 for my $d (@$line) {
  98                         if (!defined($d)) {
  99                                 $d = '\N';
 100                         } else {
 101                                 $d =~ s/\f/\\f/gos;
 102                                 $d =~ s/\n/\\n/gos;
 103                                 $d =~ s/\r/\\r/gos;
 104                                 $d =~ s/\t/\\t/gos;
 105                                 $d =~ s/\\/\\\\/gos;
 106                         }
 107                         if ($h eq 'bre' and $fieldcache{$h}{fields}[$x] eq 'quality') {
 108                                 $d = int($d);
 109                         }
 110                         push @data, $d;
 111                         $x++;
 112                 }
 113                 $output->print( join("\t", @data)."\n" );
 114         }
 115
 116         $output->print('\.'."\n\n");
 117
 118         $output->print("SELECT setval('$fieldcache{$h}{sequence}'::TEXT, (SELECT MAX($fieldcache{$h}{pkey}) FROM $fieldcache{$h}{table}), TRUE);\n\n")
 119                 if (!grep { $_ eq $h} @auto);
 120 }
 121
 122 $output->print("COMMIT;\n\n");
 123 $output->close;