1 package OpenILS::WWW::TemplateBatchBibUpdate;
7 use Apache2::Const -compile => qw(OK REDIRECT DECLINED NOT_FOUND :log);
8 use APR::Const -compile => qw(:error SUCCESS);
11 use Apache2::RequestRec ();
12 use Apache2::RequestIO ();
13 use Apache2::RequestUtil;
18 use OpenSRF::EX qw(:try);
19 use OpenSRF::Utils qw/:datetime/;
20 use OpenSRF::Utils::Cache;
22 use OpenSRF::AppSession;
27 use Unicode::Normalize;
28 use OpenILS::Utils::Fieldmapper;
29 use OpenSRF::Utils::Logger qw/$logger/;
32 use MARC::File::XML ( BinaryEncoding => 'UTF-8' );
34 use UNIVERSAL::require;
36 our @formats = qw/USMARC UNIMARC XML BRE/;
38 # set the bootstrap config and template include directory when
39 # this module is loaded
49 OpenSRF::System->bootstrap_client( config_file => $bootstrap );
50 Fieldmapper->import(IDL => OpenSRF::Utils::SettingsClient->new->config_value("IDL"));
51 return Apache2::Const::OK;
58 my $authid = $cgi->cookie('ses') || $cgi->param('ses');
59 my $usr = verify_login($authid);
60 return show_template($r) unless ($usr);
62 my $template = $cgi->param('template');
63 return show_template($r) unless ($template);
66 my $rsource = $cgi->param('recordSource');
70 if ($rsource eq 'r') {
71 @records = map { $_ ? ($_) : () } $cgi->param('recid');
74 if ($rsource eq 'c') { # try for a file
75 my $file = $cgi->param('idfile');
77 my $col = $cgi->param('idcolumn') || 0;
78 my $csv = new Text::CSV;
82 my @data = $csv->fields;
91 my $e = OpenSRF::AppSession->connect('open-ils.cstore');
92 $e->request('open-ils.cstore.transaction.begin')->gather(1);
93 $e->request('open-ils.cstore.set_audit_info', $authid, $usr->id, $usr->wsid)->gather(1);
95 # still no records ...
96 my $container = $cgi->param('containerid');
97 if ($rsource eq 'b') {
99 my $bucket = $e->request(
100 'open-ils.cstore.direct.container.biblio_record_entry_bucket.retrieve',
104 $e->request('open-ils.cstore.transaction.rollback')->gather(1);
106 $r->log->error("No such bucket $container");
107 $logger->error("No such bucket $container");
108 return Apache2::Const::NOT_FOUND;
110 my $recs = $e->request(
111 'open-ils.cstore.direct.container.biblio_record_entry_bucket_item.search.atomic',
112 { bucket => $container }
114 @records = map { ($_->target_biblio_record_entry) } @$recs;
119 $e->request('open-ils.cstore.transaction.rollback')->gather(1);
121 return show_template($r);
124 # we have a template and some record ids, so...
126 # insert the template record
127 my $min_id = $e->request(
128 'open-ils.cstore.json_query',
129 { select => { bre => [{ column => 'id', transform => 'min', aggregate => 1}] }, from => 'bre' }
130 )->gather(1)->{id} - 1;
132 warn "new template bib id = $min_id\n";
134 my $tmpl_rec = Fieldmapper::biblio::record_entry->new;
135 $tmpl_rec->id($min_id);
136 $tmpl_rec->deleted('t');
137 $tmpl_rec->active('f');
138 $tmpl_rec->marc($template);
139 $tmpl_rec->creator($usr->id);
140 $tmpl_rec->editor($usr->id);
142 warn "about to create bib $min_id\n";
143 $e->request('open-ils.cstore.direct.biblio.record_entry.create', $tmpl_rec )->gather(1);
145 # create the new container for the records and the template
146 my $bucket = Fieldmapper::container::biblio_record_entry_bucket->new;
147 $bucket->owner($usr->id);
148 $bucket->btype('template_merge');
150 my $bname = $cgi->param('bname') || 'Temporary Merge Bucket '. localtime() . ' ' . $usr->id;
151 $bucket->name($bname);
153 $bucket = $e->request('open-ils.cstore.direct.container.biblio_record_entry_bucket.create', $bucket )->gather(1);
155 # create items in the bucket
156 my $item = Fieldmapper::container::biblio_record_entry_bucket_item->new;
157 $item->bucket($bucket->id);
158 $item->target_biblio_record_entry($min_id);
160 $e->request('open-ils.cstore.direct.container.biblio_record_entry_bucket_item.create', $item )->gather(1);
163 for my $r (@records) {
165 $item->target_biblio_record_entry($r);
166 $e->request('open-ils.cstore.direct.container.biblio_record_entry_bucket_item.create', $item )->gather(1);
170 $e->request('open-ils.cstore.transaction.commit')->gather(1);
173 # fire the background bucket processor
174 my $cache_key = OpenSRF::AppSession
175 ->create('open-ils.cat')
176 ->request('open-ils.cat.container.template_overlay.background', $authid, $bucket->id)
179 return show_processing_template($r, $bucket->id, \@records, $cache_key);
183 my $auth_token = shift;
184 return undef unless $auth_token;
186 my $user = OpenSRF::AppSession
187 ->create("open-ils.auth")
188 ->request( "open-ils.auth.session.retrieve", $auth_token )
191 if (ref($user) eq 'HASH' && $user->{ilsevent} == 1001) {
195 return $user if ref($user);
199 sub show_processing_template {
203 my $cache_key = shift;
205 my $rec_string = @$recs;
207 $r->content_type('text/html');
209 <html xmlns="http://www.w3.org/1999/xhtml">
212 <title>Merging records...</title>
213 <style type="text/css">
214 \@import '/js/dojo/dojo/resources/dojo.css';
215 \@import '/js/dojo/dijit/themes/tundra/tundra.css';
216 .hide_me { display: none; visibility: hidden; }
217 th { font-weight: bold; }
220 <script type="text/javascript">
224 AutoIDL: ['aou','aout','pgt','au','cbreb']
228 <script src='/js/dojo/dojo/dojo.js'></script>
229 <!-- <script src="/js/dojo/dojo/openils_dojo.js"></script> -->
231 <script type="text/javascript">
233 dojo.require('fieldmapper.AutoIDL');
234 dojo.require('fieldmapper.dojoData');
235 dojo.require('openils.User');
236 dojo.require('openils.XUL');
237 dojo.require('dojo.cookie');
238 dojo.require('openils.CGI');
239 dojo.require('openils.widget.ProgressDialog');
241 var cgi = new openils.CGI();
242 var authtoken = dojo.cookie('ses') || cgi.param('ses');
243 if (!authtoken && openils.XUL.isXUL()) {
244 var stash = openils.XUL.getStash();
245 authtoken = stash.session.key;
247 var u = new openils.User({ authtoken: authtoken });
249 dojo.addOnLoad(function () {
250 progress_dialog.show(true);
251 progress_dialog.update({maximum:$rec_string});
254 interval = setInterval( function() {
255 fieldmapper.standardRequest(
256 ['open-ils.actor','open-ils.actor.anon_cache.get_value'],
258 params: [ u.authtoken, 'res_list' ],
259 onerror : function (r) { progress_dialog.hide(); },
260 onresponse : function (r) {
261 var counter = { success : 0, fail : 0, total : 0 };
262 dojo.forEach( openils.Util.readResponse(r), function(x) {
264 clearInterval(interval);
265 progress_dialog.hide();
266 if (x.success == 't') dojo.byId('complete_msg').innerHTML = 'Overlay completed successfully';
267 else dojo.byId('complete_msg').innerHTML = 'Overlay did not complet successfully';
281 // update the progress dialog
282 progress_dialog.update({progress:counter.total});
283 dojo.byId('success_count').innerHTML = counter.success;
284 dojo.byId('fail_count').innerHTML = counter.fail;
285 dojo.byId('total_count').innerHTML = counter.total;
295 <body style="margin:10px;" class='tundra'>
296 <div class="hide_me"><div dojoType="openils.widget.ProgressDialog" jsId="progress_dialog"></div></div>
298 <table style="width:100%; margin-top:100px;">
301 <td>Record Count</td>
305 <td id='success_count'></td>
309 <td id='fail_count'></td>
313 <td id='total_count'></td>
317 <div id='complete_msg'></div>
323 return Apache2::Const::OK;
330 $r->content_type('text/html');
332 <html xmlns="http://www.w3.org/1999/xhtml">
335 <title>Merge Template Builder</title>
336 <style type="text/css">
337 @import '/js/dojo/dojo/resources/dojo.css';
338 @import '/js/dojo/dijit/themes/tundra/tundra.css';
339 .hide_me { display: none; visibility: hidden; }
340 table.ruleTable th { padding: 5px; border-collapse: collapse; border-bottom: solid 1px gray; font-weight: bold; }
341 table.ruleTable td { padding: 5px; border-collapse: collapse; border-bottom: solid 1px gray; }
344 <script type="text/javascript">
348 AutoIDL: ['aou','aout','pgt','au','cbreb']
352 <script src='/js/dojo/dojo/dojo.js'></script>
353 <!-- <script src="/js/dojo/dojo/openils_dojo.js"></script> -->
355 <script type="text/javascript">
357 dojo.require('dojo.data.ItemFileReadStore');
358 dojo.require('dijit.form.Form');
359 dojo.require('dijit.form.NumberSpinner');
360 dojo.require('dijit.form.FilteringSelect');
361 dojo.require('dijit.form.TextBox');
362 dojo.require('dijit.form.Textarea');
363 dojo.require('dijit.form.Button');
364 dojo.require('MARC.Batch');
365 dojo.require('fieldmapper.AutoIDL');
366 dojo.require('fieldmapper.dojoData');
367 dojo.require('openils.User');
368 dojo.require('openils.CGI');
369 dojo.require('openils.XUL');
370 dojo.require('dojo.cookie');
372 var cgi = new openils.CGI();
373 var authtoken = dojo.cookie('ses') || cgi.param('ses');
374 if (!authtoken && openils.XUL.isXUL()) {
375 var stash = openils.XUL.getStash();
376 authtoken = stash.session.key;
378 var u = new openils.User({ authtoken: authtoken });
380 var bucketStore = new dojo.data.ItemFileReadStore(
381 { data : cbreb.toStoreData(
382 fieldmapper.standardRequest(
383 ['open-ils.actor','open-ils.actor.container.retrieve_by_class.authoritative'],
384 [u.authtoken, u.user.id(), 'biblio', 'staff_client']
390 function render_preview () {
391 var rec = ruleset_to_record();
392 dojo.byId('marcPreview').innerHTML = rec.toBreaker();
395 function render_from_template () {
396 var kid_number = dojo.byId('ruleList').childNodes.length;
397 var clone = dojo.query('*[name=ruleTable]', dojo.byId('ruleTemplate'))[0].cloneNode(true);
399 var typeSelect = dojo.query('*[name=typeSelect]',clone).instantiate(dijit.form.FilteringSelect, {
400 onChange : function (val) {
404 dijit.byNode(dojo.query('*[name=marcDataContainer] .dijit',clone)[0]).attr('disabled',false);
407 dijit.byNode(dojo.query('*[name=marcDataContainer] .dijit',clone)[0]).attr('disabled',true);
413 var marcData = dojo.query('*[name=marcData]',clone).instantiate(dijit.form.TextBox, {
414 onChange : render_preview
418 var tag = dojo.query('*[name=tag]',clone).instantiate(dijit.form.TextBox, {
419 onChange : function (newtag) {
420 var md = dijit.byNode(dojo.query('*[name=marcDataContainer] .dijit',clone)[0]);
421 var current_marc = md.attr('value');
423 if (newtag.length == 3) {
424 if (current_marc.length == 0) newtag += ' \\\\';
425 if (current_marc.substr(0,3) != newtag) current_marc = newtag + current_marc.substr(3);
427 md.attr('value', current_marc);
432 var sf = dojo.query('*[name=sf]',clone).instantiate(dijit.form.TextBox, {
433 onChange : function (newsf) {
434 var md = dijit.byNode(dojo.query('*[name=marcDataContainer] .dijit',clone)[0]);
435 var current_marc = md.attr('value');
436 var sf_list = newsf.split('');
438 for (var i in sf_list) {
439 var re = '\\$' + sf_list[i];
440 if (current_marc.match(re)) continue;
441 current_marc += '$' + sf_list[i];
444 md.attr('value', current_marc);
449 var matchSF = dojo.query('*[name=matchSF]',clone).instantiate(dijit.form.TextBox, {
450 onChange : render_preview
453 var matchRE = dojo.query('*[name=matchRE]',clone).instantiate(dijit.form.TextBox, {
454 onChange : render_preview
457 var removeButton = dojo.query('*[name=removeButton]',clone).instantiate(dijit.form.Button, {
458 onClick : function() {
460 dojo.byId('ruleList').childNodes[kid_number],
467 dojo.place(clone,'ruleList');
470 function ruleset_to_record () {
471 var rec = new MARC.Record ({ delimiter : '$' });
474 dojo.query('#ruleList *[name=ruleTable]').filter( function (node) {
475 if (node.className.match(/hide_me/)) return false;
479 var rule_tag = new MARC.Field ({
484 var rule_txt = dijit.byNode(dojo.query('*[name=tagContainer] .dijit',tbl)[0]).attr('value');
485 rule_txt += dijit.byNode(dojo.query('*[name=sfContainer] .dijit',tbl)[0]).attr('value');
487 var reSF = dijit.byNode(dojo.query('*[name=matchSFContainer] .dijit',tbl)[0]).attr('value');
489 var reRE = dijit.byNode(dojo.query('*[name=matchREContainer] .dijit',tbl)[0]).attr('value');
490 rule_txt += '[' + reSF + '~' + reRE + ']';
493 var rtype = dijit.byNode(dojo.query('*[name=typeSelectContainer] .dijit',tbl)[0]).attr('value');
494 rule_tag.addSubfields( rtype, rule_txt )
495 rec.appendFields( rule_tag );
497 if (rtype == 'a' || rtype == 'r') {
501 marcbreaker : dijit.byNode(dojo.query('*[name=marcDataContainer] .dijit',tbl)[0]).attr('value')
513 <body style="margin:10px;" class='tundra'>
515 <div dojoType="dijit.form.Form" id="myForm" jsId="myForm" encType="multipart/form-data" action="" method="POST">
516 <script type='dojo/method' event='onSubmit'>
517 var rec = ruleset_to_record();
519 if (rec.subfield('905','r') == '') { // no-op to force replace mode
525 subfields : [['r','901c']]
530 dojo.byId('template_value').value = rec.toXmlString();
534 <input type='hidden' id='template_value' name='template'/>
536 <label for='inputTypeSelect'>Record source:</label>
537 <select name='recordSource' dojoType='dijit.form.FilteringSelect'>
538 <script type='dojo/method' event='onChange' args="val">
541 dojo.removeClass('bucketListContainer','hide_me');
542 dojo.addClass('csvContainer','hide_me');
543 dojo.addClass('recordContainer','hide_me');
546 dojo.addClass('bucketListContainer','hide_me');
547 dojo.removeClass('csvContainer','hide_me');
548 dojo.addClass('recordContainer','hide_me');
551 dojo.addClass('bucketListContainer','hide_me');
552 dojo.addClass('csvContainer','hide_me');
553 dojo.removeClass('recordContainer','hide_me');
557 <script type='dojo/method' event='postCreate'>
558 if (cgi.param('recordSource')) {
559 this.attr('value',cgi.param('recordSource'));
560 this.onChange(cgi.param('recordSource'));
563 <option value='b'>a Bucket</option>
564 <option value='c'>a CSV File</option>
565 <option value='r'>a specific record ID</option>
568 <table style='margin:10px; margin-bottom:20px;'>
571 <th>Merge template name (optional):</th>
572 <td><input id='bucketName' jsId='bucketName' type='text' dojoType='dijit.form.TextBox' name='bname' value=''/></td>
575 <tr class='' id='bucketListContainer'>
577 <div name='containerid' jsId='bucketList' dojoType='dijit.form.FilteringSelect' store='bucketStore' searchAttr='name' id='bucketList'>
578 <script type='dojo/method' event='postCreate'>
579 if (cgi.param('containerid')) this.attr('value',cgi.param('containerid'));
584 <tr class='hide_me' id='csvContainer'>
586 Column <input style='width:75px;' type='text' dojoType='dijit.form.NumberSpinner' name='idcolumn' value='0' constraints='{min:0,max:100,places:0}' /> of:
587 <input id='idfile' type="file" name="idfile"/>
590 Columns are numbered starting at 0. For instance, when looking at a CSV file in Excel, the column labeled A is the same as column 0, and the column labeled B is the same as column 1.
593 <tr class='hide_me' id='recordContainer'>
594 <td>Record ID: <input dojoType='dijit.form.TextBox' name='recid' style='width:75px;' type='text' value=''/></td>
598 <button type="submit" dojoType='dijit.form.Button'>GO!</button> (After setting up your template below.)
603 </div> <!-- end of the form -->
606 <table style='width: 100%'>
608 <td style='width: 50%'><div id='ruleList'></div></td>
609 <td valign='top'>Update Template Preview:<br/><pre id='marcPreview'></pre></td>
613 <button dojoType='dijit.form.Button'>Add Merge Rule
614 <script type='dojo/connect' event='onClick'>render_from_template()</script>
615 <script type='dojo/method' event='postCreate'>render_from_template()</script>
618 <div class='hide_me' id='ruleTemplate'>
619 <div name='ruleTable'>
620 <table class='ruleTable'>
623 <th style="text-align:center;">Rule Setup</th>
624 <th style="text-align:center;">Data</th>
625 <th style="text-align:center;">Help</th>
628 <th>Action (Rule Type)</th>
629 <td name='typeSelectContainer'>
630 <select name='typeSelect'>
631 <option value='r'>Replace</option>
632 <option value='a'>Add</option>
633 <option value='d'>Delete</option>
636 <td>How to change the existing records</td>
640 <td name='tagContainer'><input style='with: 2em;' name='tag' type='text'></input</td>
641 <td>Three characters, no spaces, no indicators, etc. eg: 245</td>
644 <th>Subfields (optional)</th>
645 <td name='sfContainer'><input name='sf' type='text'/></td>
646 <td>No spaces, no delimiters, eg: abcnp</td>
650 <td name='marcDataContainer'><input name='marcData' type='text'/></td>
651 <td>MARC-Breaker formatted data with indicators and subfield delimiters, eg:<br/>245 04$aThe End</td>
654 <th colspan='3' style='padding-top: 20px; text-align: center;'>Advanced Matching Restriction (Optional)</th>
658 <td name='matchSFContainer'><input style='with: 2em;' name='matchSF' type='text'></input</td>
659 <td>A single subfield code, no delimiters, eg: a</td>
661 <th>Regular Expression</th>
662 <td name='matchREContainer'><input name='matchRE' type='text'/></td>
663 <td>See <a href="http://perldoc.perl.org/perlre.html#Regular-Expressions" target="_blank">the Perl documentation</a>
664 for an explanation of Regular Expressions.
668 <td colspan='3' style='padding-top: 20px; text-align: center;'>
669 <button name='removeButton'>Remove this Template Rule</button>
682 return Apache2::Const::OK;