spool processing
[Evergreen.git] / Open-ILS / src / perlmods / OpenILS / Application / Vandelay.pm
1 package OpenILS::Application::Vandelay;
2 use OpenILS::Application;
3 use base qw/OpenILS::Application/;
4
5 use Unicode::Normalize;
6 use OpenSRF::EX qw/:try/;
7
8 use OpenSRF::AppSession;
9 use OpenSRF::Utils::SettingsClient;
10 use OpenSRF::Utils::Cache;
11
12 use OpenILS::Utils::Fieldmapper;
13 use OpenILS::Utils::CStoreEditor qw/:funcs/;
14
15 use MARC::Batch;
16 use MARC::Record;
17 use MARC::File::XML;
18
19 use OpenILS::Utils::Fieldmapper;
20
21 use Time::HiRes qw(time);
22
23 use OpenSRF::Utils::Logger qw/:level/;
24 my $log = 'OpenSRF::Utils::Logger';
25
26 sub initialize {}
27 sub child_init {}
28
29 sub entityize {
30         my $self = shift;
31         my $stuff = shift;
32         my $form = shift;
33
34         if ($form eq 'D') {
35                 $stuff = NFD($stuff);
36         } else {
37                 $stuff = NFC($stuff);
38         }
39
40         $stuff =~ s/([\x{0080}-\x{fffd}])/sprintf('&#x%X;',ord($1))/sgoe;
41         return $stuff;
42 }
43
44 # --------------------------------------------------------------------------------
45 # Biblio ingest
46
47 sub create_bib_queue {
48         my $self = shift;
49         my $client = shift;
50         my $auth = shift;
51         my $name = shift;
52         my $owner = shift;
53         my $type = shift;
54         my $purpose = shift;
55
56         my $e = new_editor(authtoken => $auth, xact => 1);
57
58         return $e->die_event unless $e->checkauth;
59         return $e->die_event unless $e->allowed('CREATE_BIB_IMPORT_QUEUE', $owner);
60
61         my $queue = new Fieldmapper::vandelay::bib_queue();
62         $queue->name( $name );
63         $queue->owner( $owner );
64         $queue->queue_type( $type ) if ($type);
65         $queue->queue_purpose( $purpose ) if ($purpose);
66
67         my $new_id = $e->create_vandelay_bib_queue( $queue );
68         $e->die_event unless ($new_id);
69         $e->commit;
70
71         $queue->id($new_id);
72         return $queue;
73 }
74 __PACKAGE__->register_method(  
75         api_name        => "open-ils.vandelay.bib_queue.create",
76         method          => "create_bib_queue",
77         api_level       => 1,
78         argc            => 3,
79 );                      
80
81
82 sub create_auth_queue {
83         my $self = shift;
84         my $client = shift;
85         my $auth = shift;
86         my $name = shift;
87         my $owner = shift;
88         my $type = shift;
89         my $purpose = shift;
90
91         my $e = new_editor(authtoken => $auth, xact => 1);
92
93         return $e->die_event unless $e->checkauth;
94         return $e->die_event unless $e->allowed('CREATE_AUTHORITY_IMPORT_QUEUE', $owner);
95
96         my $queue = new Fieldmapper::vandelay::authority_queue();
97         $queue->name( $name );
98         $queue->owner( $owner );
99         $queue->queue_type( $type ) if ($type);
100         $queue->queue_purpose( $purpose ) if ($purpose);
101
102         my $new_id = $e->create_vandelay_authority_queue( $queue );
103         $e->die_event unless ($new_id);
104         $e->commit;
105
106         $queue->id($new_id);
107         return $queue;
108 }
109 __PACKAGE__->register_method(  
110         api_name        => "open-ils.vandelay.authority_queue.create",
111         method          => "create_auth_queue",
112         api_level       => 1,
113         argc            => 3,
114 );                      
115
116 sub add_record_to_bib_queue {
117         my $self = shift;
118         my $client = shift;
119         my $auth = shift;
120         my $queue = shift;
121         my $marc = shift;
122
123         my $e = new_editor(authtoken => $auth, xact => 1);
124
125         $queue = $e->retrieve_vandelay_bib_queue($queue);
126
127         return $e->die_event unless $e->checkauth;
128         return $e->die_event unless
129                 ($e->allowed('CREATE_BIB_IMPORT_QUEUE', undef, $queue) ||
130                  $e->allowed('CREATE_BIB_IMPORT_QUEUE', $queue->owner));
131
132         my $new_id = _add_auth_rec($e, $marc, $queue->id);
133
134         $e->die_event unless ($new_id);
135         $e->commit;
136
137         $rec->id($new_id);
138         return $rec;
139 }
140 __PACKAGE__->register_method(  
141         api_name        => "open-ils.vandelay.queued_bib_record.create",
142         method          => "add_record_to_bib_queue",
143         api_level       => 1,
144         argc            => 3,
145 );                      
146
147 sub _add_bib_rec {
148         my $e = shift;
149         my $marc = shift;
150         my $queue = shift;
151
152         my $rec = new Fieldmapper::vandelay::queued_bib_record();
153         $rec->marc( $marc );
154         $rec->queue( $queue );
155
156         return $e->create_vandelay_queued_bib_record( $rec );
157 }
158
159 sub add_record_to_authority_queue {
160         my $self = shift;
161         my $client = shift;
162         my $auth = shift;
163         my $queue = shift;
164         my $marc = shift;
165
166         my $e = new_editor(authtoken => $auth, xact => 1);
167
168         $queue = $e->retrieve_vandelay_authority_queue($queue);
169
170         return $e->die_event unless $e->checkauth;
171         return $e->die_event unless
172                 ($e->allowed('CREATE_AUTHORITY_IMPORT_QUEUE', undef, $queue) ||
173                  $e->allowed('CREATE_AUTHORITY_IMPORT_QUEUE', $queue->owner));
174
175         my $new_id = _add_auth_rec($e, $marc, $queue->id);
176
177         $e->die_event unless ($new_id);
178         $e->commit;
179
180         $rec->id($new_id);
181         return $rec;
182 }
183 __PACKAGE__->register_method(
184         api_name        => "open-ils.vandelay.queued_authority_record.create",
185         method          => "add_record_to_authority_queue",
186         api_level       => 1,
187         argc            => 3,
188 );
189
190 sub _add_auth_rec {
191         my $e = shift;
192         my $marc = shift;
193         my $queue = shift;
194
195         my $rec = new Fieldmapper::vandelay::queued_authority_record();
196         $rec->marc( $marc );
197         $rec->queue( $queue );
198
199         return $e->create_vandelay_queued_authority_record( $rec );
200 }
201
202 sub process_spool {
203         my $self = shift;
204         my $client = shift;
205         my $auth = shift;
206         my $fingerprint = shift;
207         my $queue = shift;
208
209         my $e = new_editor(authtoken => $auth, xact => 1);
210
211         if ($self->{record_type} eq 'bib') {
212                 return $e->die_event unless $e->checkauth;
213                 return $e->die_event unless
214                         ($e->allowed('CREATE_BIB_IMPORT_QUEUE', undef, $queue) ||
215                          $e->allowed('CREATE_BIB_IMPORT_QUEUE', $queue->owner));
216         } else {
217                 return $e->die_event unless $e->checkauth;
218                 return $e->die_event unless
219                         ($e->allowed('CREATE_AUTHORITY_IMPORT_QUEUE', undef, $queue) ||
220                          $e->allowed('CREATE_AUTHORITY_IMPORT_QUEUE', $queue->owner));
221         }
222
223         my $method = 'open-ils.vandelay.queued_'.$self->{record_type}.'_record.create';
224         $method = $self->method_lookup( $method );
225
226     my $cache = new OpenSRF::Utils::Cache();
227
228     my $data = $cache->get_cache('vandelay_import_spool_' . $fingerprint);
229     $data = decode_base64($data);
230
231         my $fh = new IO::Scalar \$data;
232
233         my $batch = new MARC::Batch ( $type, $fh );
234         $batch->strict_off;
235
236         my $count = 0;
237         while (my $r = $batch->next) {
238                 try {
239                         (my $xml = $rec->as_xml_record()) =~ s/\n//sog;
240                         $xml =~ s/^<\?xml.+\?\s*>//go;
241                         $xml =~ s/>\s+</></go;
242                         $xml =~ s/\p{Cc}//go;
243                         $xml = $self->entityize($xml);
244                         $xml =~ s/[\x00-\x1f]//go;
245
246                         if ($self->{record_type} eq 'bib') {
247                                 _add_bib_rec( $e, $xml, $queue );
248                         } else {
249                                 _add_auth_rec( $e, $xml, $queue );
250                         }
251                         $count++;
252                         
253                         $client->respond( $count );
254                 } catch Error with {
255                         my $error = shift;
256                         $log->warn("Encountered a bad record at Vandelay ingest: ".$error);
257                 }
258         }
259
260         $e->commit;
261         return undef;
262 }
263 __PACKAGE__->register_method(  
264         api_name        => "open-ils.vandelay.bib.process_spool",
265         method          => "process_spool",
266         api_level       => 1,
267         argc            => 3,
268         record_type     => 'bib'
269 );                      
270 __PACKAGE__->register_method(  
271         api_name        => "open-ils.vandelay.auth.process_spool",
272         method          => "process_spool",
273         api_level       => 1,
274         argc            => 3,
275         record_type     => 'auth'
276 );                      
277
278
279
280 1;
281