safer MR mapping
[Evergreen.git] / Open-ILS / src / extras / import / pg_loader.pl
1 #!/usr/bin/perl
2 use strict;
3
4 use lib '/openils/lib/perl5/';
5
6 use OpenSRF::System;
7 use OpenSRF::EX qw/:try/;
8 use OpenSRF::Utils::SettingsClient;
9 use OpenILS::Utils::Fieldmapper;
10 use OpenSRF::Utils::JSON;
11 use FileHandle;
12
13 use Time::HiRes qw/time/;
14 use Getopt::Long;
15
16 my @files;
17 my ($config, $output, @auto, @order, @wipe, $quiet) =
18         ('/openils/conf/opensrf_core.xml');
19
20 GetOptions(
21         'config=s'      => \$config,
22         'output=s'      => \$output,
23         'wipe=s'        => \@wipe,
24         'autoprimary=s' => \@auto,
25         'order=s'       => \@order,
26         'quiet'         => \$quiet,
27 );
28
29 my %lineset;
30 my %fieldcache;
31
32 OpenSRF::System->bootstrap_client( config_file => $config );
33 Fieldmapper->import(IDL => OpenSRF::Utils::SettingsClient->new->config_value("IDL"));
34
35 my $count = 0;
36 my $starttime = time;
37 while ( my $rec = <> ) {
38         next unless ($rec);
39
40         my $row;
41         try {
42                 $row = OpenSRF::Utils::JSON->JSON2perl($rec);
43         } catch Error with {
44                 my $e = shift;
45                 warn "\n\n !!! Error : $e \n\n at or around line $count\n";
46         };
47         next unless ($row);
48
49         my $class = $row->class_name;
50         my $hint = $row->json_hint;
51
52         if (!$lineset{$hint}) {
53                 $lineset{$hint} = [];
54                 my @cols = $row->real_fields;
55                 if (grep { $_ eq $hint} @auto) {
56                         @cols = grep { $_ ne $class->Identity } @cols;
57                 }
58
59                 $fieldcache{$hint} =
60                         { table => $class->Table,
61                           sequence => $class->Sequence,
62                           pkey => $class->Identity,
63                           fields => \@cols,
64                         };
65
66         #XXX it burnnnsssessss
67         $fieldcache{$hint}{table} =~ s/\.full_rec/.real_full_rec/o if ($hint eq 'mfr');
68         }
69
70         push @{ $lineset{$hint} }, [map { $row->$_ } @{ $fieldcache{$hint}{fields} }];
71
72         if (!$quiet && !($count % 500)) {
73                 print STDERR "\r$count\t". $count / (time - $starttime);
74         }
75
76         $count++;
77 }
78
79 print STDERR "\nWriting file ...\n" if (!$quiet);
80
81 $output = '&STDOUT' unless ($output);
82 $output = FileHandle->new(">$output") if ($output);
83
84 binmode($output,'utf8');
85
86 $output->print("SET CLIENT_ENCODING TO 'UNICODE';\n\n");
87 $output->print("BEGIN;\n\n");
88
89 for my $h (@order) {
90         # continue if there was no data for this table
91         next unless ($fieldcache{$h});
92
93         my $fields = join(',', @{ $fieldcache{$h}{fields} });
94         $output->print( "DELETE FROM $fieldcache{$h}{table};\n" ) if (grep {$_ eq $h } @wipe);
95         # Speed up loading of bib records
96         if ($h eq 'mfr') {
97                 $output->print("\nSELECT reporter.disable_materialized_simple_record_trigger();\n");
98         }
99         $output->print( "COPY $fieldcache{$h}{table} ($fields) FROM STDIN;\n" );
100
101         for my $line (@{ $lineset{$h} }) {
102                 my @data;
103                 my $x = 0;
104                 for my $d (@$line) {
105                         if (!defined($d)) {
106                                 $d = '\N';
107                         } else {
108                                 $d =~ s/\f/\\f/gos;
109                                 $d =~ s/\n/\\n/gos;
110                                 $d =~ s/\r/\\r/gos;
111                                 $d =~ s/\t/\\t/gos;
112                                 $d =~ s/\\/\\\\/gos;
113                         }
114                         if ($h eq 'bre' and $fieldcache{$h}{fields}[$x] eq 'quality') {
115                                 $d = int($d);
116                         }
117                         push @data, $d;
118                         $x++;
119                 }
120                 $output->print( join("\t", @data)."\n" );
121         }
122
123         $output->print('\.'."\n\n");
124         
125         if ($h eq 'mfr') {
126                 $output->print("SELECT reporter.enable_materialized_simple_record_trigger();\n");
127         }
128
129         $output->print("SELECT setval('$fieldcache{$h}{sequence}'::TEXT, (SELECT MAX($fieldcache{$h}{pkey}) FROM $fieldcache{$h}{table}), TRUE);\n\n")
130                 if (!grep { $_ eq $h} @auto);
131 }
132
133 $output->print("COMMIT;\n\n");
134 $output->close;