1 package OpenILS::WWW::TemplateBatchBibUpdate;
7 use Apache2::Const -compile => qw(OK REDIRECT DECLINED NOT_FOUND :log);
8 use APR::Const -compile => qw(:error SUCCESS);
11 use Apache2::RequestRec ();
12 use Apache2::RequestIO ();
13 use Apache2::RequestUtil;
18 use OpenSRF::EX qw(:try);
19 use OpenILS::Utils::DateTime qw/:datetime/;
20 use OpenSRF::Utils::Cache;
22 use OpenSRF::AppSession;
27 use Unicode::Normalize;
28 use OpenILS::Utils::Fieldmapper;
29 use OpenSRF::Utils::Logger qw/$logger/;
32 use MARC::File::XML ( BinaryEncoding => 'UTF-8' );
34 use UNIVERSAL::require;
36 our @formats = qw/USMARC UNIMARC XML BRE/;
38 # set the bootstrap config and template include directory when
39 # this module is loaded
49 OpenSRF::System->bootstrap_client( config_file => $bootstrap );
50 Fieldmapper->import(IDL => OpenSRF::Utils::SettingsClient->new->config_value("IDL"));
51 return Apache2::Const::OK;
58 my $authid = $cgi->cookie('ses') || $cgi->param('ses');
59 my $usr = verify_login($authid);
60 return show_template($r) unless ($usr);
62 my $template = $cgi->param('template');
63 return show_template($r) unless ($template);
66 my $rsource = $cgi->param('recordSource');
70 if ($rsource eq 'r') {
71 @records = map { $_ ? ($_) : () } $cgi->param('recid');
74 if ($rsource eq 'c') { # try for a file
75 my $file = $cgi->param('idfile');
77 my $col = $cgi->param('idcolumn') || 0;
78 my $csv = new Text::CSV;
82 my @data = $csv->fields;
91 my $e = OpenSRF::AppSession->connect('open-ils.cstore');
92 $e->request('open-ils.cstore.transaction.begin')->gather(1);
93 $e->request('open-ils.cstore.set_audit_info', $authid, $usr->id, $usr->wsid)->gather(1);
95 # still no records ...
96 my $container = $cgi->param('containerid');
97 if ($rsource eq 'b') {
99 my $bucket = $e->request(
100 'open-ils.cstore.direct.container.biblio_record_entry_bucket.retrieve',
104 $e->request('open-ils.cstore.transaction.rollback')->gather(1);
106 $r->log->error("No such bucket $container");
107 $logger->error("No such bucket $container");
108 return Apache2::Const::NOT_FOUND;
110 my $recs = $e->request(
111 'open-ils.cstore.direct.container.biblio_record_entry_bucket_item.search.atomic',
112 { bucket => $container }
114 @records = map { ($_->target_biblio_record_entry) } @$recs;
119 $e->request('open-ils.cstore.transaction.rollback')->gather(1);
121 return show_template($r);
124 # we have a template and some record ids, so...
126 # insert the template record
127 my $min_id = $e->request(
128 'open-ils.cstore.json_query',
129 { select => { bre => [{ column => 'id', transform => 'min', aggregate => 1}] }, from => 'bre' }
130 )->gather(1)->{id} - 1;
132 warn "new template bib id = $min_id\n";
134 my $tmpl_rec = Fieldmapper::biblio::record_entry->new;
135 $tmpl_rec->id($min_id);
136 $tmpl_rec->deleted('t');
137 $tmpl_rec->active('f');
138 $tmpl_rec->marc($template);
139 $tmpl_rec->creator($usr->id);
140 $tmpl_rec->editor($usr->id);
142 warn "about to create bib $min_id\n";
143 $e->request('open-ils.cstore.direct.biblio.record_entry.create', $tmpl_rec )->gather(1);
145 # create the new container for the records and the template
146 my $bucket = Fieldmapper::container::biblio_record_entry_bucket->new;
147 $bucket->owner($usr->id);
148 $bucket->btype('template_merge');
150 my $bname = $cgi->param('bname') || 'Temporary Merge Bucket '. localtime() . ' ' . $usr->id;
151 $bucket->name($bname);
153 $bucket = $e->request('open-ils.cstore.direct.container.biblio_record_entry_bucket.create', $bucket )->gather(1);
155 # create items in the bucket
156 my $item = Fieldmapper::container::biblio_record_entry_bucket_item->new;
157 $item->bucket($bucket->id);
158 $item->target_biblio_record_entry($min_id);
160 $e->request('open-ils.cstore.direct.container.biblio_record_entry_bucket_item.create', $item )->gather(1);
163 for my $r (@records) {
165 $item->target_biblio_record_entry($r);
166 $e->request('open-ils.cstore.direct.container.biblio_record_entry_bucket_item.create', $item )->gather(1);
170 $e->request('open-ils.cstore.transaction.commit')->gather(1);
173 # fire the background bucket processor
174 my $cache_key = OpenSRF::AppSession
175 ->create('open-ils.cat')
176 ->request('open-ils.cat.container.template_overlay.background', $authid, $bucket->id)
179 return show_processing_template($r, $bucket->id, \@records, $cache_key);
183 my $auth_token = shift;
184 return undef unless $auth_token;
186 my $user = OpenSRF::AppSession
187 ->create("open-ils.auth")
188 ->request( "open-ils.auth.session.retrieve", $auth_token )
191 if (ref($user) eq 'HASH' && $user->{ilsevent} == 1001) {
195 return $user if ref($user);
199 sub show_processing_template {
203 my $cache_key = shift;
205 my $rec_string = @$recs;
207 $r->content_type('text/html');
209 <html xmlns="http://www.w3.org/1999/xhtml">
212 <title>Merging records...</title>
213 <style type="text/css">
214 \@import '/js/dojo/dojo/resources/dojo.css';
215 \@import '/js/dojo/dijit/themes/tundra/tundra.css';
216 .hide_me { display: none; visibility: hidden; }
217 th { font-weight: bold; }
220 <script type="text/javascript">
224 AutoIDL: ['aou','aout','pgt','au','cbreb']
228 <script src='/js/dojo/dojo/dojo.js'></script>
229 <!-- <script src="/js/dojo/dojo/openils_dojo.js"></script> -->
231 <script type="text/javascript">
233 dojo.require('fieldmapper.AutoIDL');
234 dojo.require('fieldmapper.dojoData');
235 dojo.require('openils.User');
236 dojo.require('openils.XUL');
237 dojo.require('dojo.cookie');
238 dojo.require('openils.CGI');
239 dojo.require('openils.widget.ProgressDialog');
241 var cgi = new openils.CGI();
242 var authtoken = dojo.cookie('ses') || cgi.param('ses');
243 if (!authtoken && openils.XUL.isXUL()) {
244 var stash = openils.XUL.getStash();
245 authtoken = stash.session.key;
247 var u = new openils.User({ authtoken: authtoken });
249 dojo.addOnLoad(function () {
251 progress_dialog.update({maximum: $rec_string});
252 progress_dialog.attr("title", "MARC Batch Editor Progress......");
253 progress_dialog.show();
256 interval = setInterval( function() {
257 fieldmapper.standardRequest(
258 ['open-ils.actor','open-ils.actor.anon_cache.get_value'],
260 params: [ u.authtoken, 'batch_edit_progress' ],
261 onerror : function (r) { progress_dialog.hide(); },
262 onresponse : function (r) {
263 var counter = { success : 0, fail : 0, total : 0 };
264 if (x = openils.Util.readResponse(r)) {
265 counter.success = x.succeeded;
266 counter.fail = x.failed;
267 counter.total = counter.success + counter.fail;
269 clearInterval(interval);
270 progress_dialog.hide();
271 if (x.success == 't') dojo.byId('complete_msg').innerHTML = 'Overlay completed successfully';
272 else dojo.byId('complete_msg').innerHTML = 'Overlay did not complet successfully';
276 // update the progress dialog
277 progress_dialog.update({progress:counter.total});
278 dojo.byId('success_count').innerHTML = counter.success;
279 dojo.byId('fail_count').innerHTML = counter.fail;
280 dojo.byId('total_count').innerHTML = counter.total;
293 border: 1px solid black;
294 border-collapse: collapse;
300 table tr:nth-child(even) {
301 background-color: #eee;
303 table tr:nth-child(odd) {
304 background-color:#fff;
307 background-color: black;
320 text-decoration: underline;
326 <body style="margin:10px;font-size: 130%" class='tundra'>
327 <div class="hide_me"><div dojoType="openils.widget.ProgressDialog" jsId="progress_dialog"></div></div>
329 <h1>MARC Batch Editor Status</h1>
334 <th>Record Count</th>
338 <td id='success_count'></td>
342 <td id='fail_count'></td>
345 <td>Total Processed</td>
346 <td id='total_count'></td>
352 <td>Total To Process</td>
357 <div id='complete_msg'></div>
363 return Apache2::Const::OK;
370 $r->content_type('text/html');
372 <html xmlns="http://www.w3.org/1999/xhtml">
375 <title>Merge Template Builder</title>
376 <style type="text/css">
377 @import '/js/dojo/dojo/resources/dojo.css';
378 @import '/js/dojo/dijit/themes/tundra/tundra.css';
379 .hide_me { display: none; visibility: hidden; }
380 table.ruleTable th { padding: 5px; border-collapse: collapse; border-bottom: solid 1px gray; font-weight: bold; }
381 table.ruleTable td { padding: 5px; border-collapse: collapse; border-bottom: solid 1px gray; }
384 <script type="text/javascript">
388 AutoIDL: ['aou','aout','pgt','au','cbreb']
392 <script src='/js/dojo/dojo/dojo.js'></script>
393 <!-- <script src="/js/dojo/dojo/openils_dojo.js"></script> -->
395 <script type="text/javascript">
397 dojo.require('dojo.data.ItemFileReadStore');
398 dojo.require('dijit.form.Form');
399 dojo.require('dijit.form.NumberSpinner');
400 dojo.require('dijit.form.FilteringSelect');
401 dojo.require('dijit.form.TextBox');
402 dojo.require('dijit.form.Textarea');
403 dojo.require('dijit.form.Button');
404 dojo.require('MARC.Batch');
405 dojo.require('fieldmapper.AutoIDL');
406 dojo.require('fieldmapper.dojoData');
407 dojo.require('openils.User');
408 dojo.require('openils.CGI');
409 dojo.require('openils.XUL');
410 dojo.require('dojo.cookie');
412 var cgi = new openils.CGI();
413 var authtoken = dojo.cookie('ses') || cgi.param('ses');
414 if (!authtoken && openils.XUL.isXUL()) {
415 var stash = openils.XUL.getStash();
416 authtoken = stash.session.key;
418 var u = new openils.User({ authtoken: authtoken });
420 var bucketStore = new dojo.data.ItemFileReadStore(
421 { data : cbreb.toStoreData(
422 fieldmapper.standardRequest(
423 ['open-ils.actor','open-ils.actor.container.retrieve_by_class.authoritative'],
424 [u.authtoken, u.user.id(), 'biblio', 'staff_client']
430 function render_preview () {
431 var rec = ruleset_to_record();
432 dojo.byId('marcPreview').innerHTML = rec.toBreaker();
435 function render_from_template () {
436 var kid_number = dojo.byId('ruleList').childNodes.length;
437 var clone = dojo.query('*[name=ruleTable]', dojo.byId('ruleTemplate'))[0].cloneNode(true);
439 var typeSelect = dojo.query('*[name=typeSelect]',clone).instantiate(dijit.form.FilteringSelect, {
440 onChange : function (val) {
444 dijit.byNode(dojo.query('*[name=marcDataContainer] .dijit',clone)[0]).attr('disabled',false);
447 dijit.byNode(dojo.query('*[name=marcDataContainer] .dijit',clone)[0]).attr('disabled',true);
453 var marcData = dojo.query('*[name=marcData]',clone).instantiate(dijit.form.TextBox, {
454 onChange : render_preview
458 var tag = dojo.query('*[name=tag]',clone).instantiate(dijit.form.TextBox, {
459 onChange : function (newtag) {
460 var md = dijit.byNode(dojo.query('*[name=marcDataContainer] .dijit',clone)[0]);
461 var current_marc = md.attr('value');
463 if (newtag.length == 3) {
464 if (current_marc.length == 0) newtag += ' \\\\';
465 if (current_marc.substr(0,3) != newtag) current_marc = newtag + current_marc.substr(3);
467 md.attr('value', current_marc);
472 var sf = dojo.query('*[name=sf]',clone).instantiate(dijit.form.TextBox, {
473 onChange : function (newsf) {
474 var md = dijit.byNode(dojo.query('*[name=marcDataContainer] .dijit',clone)[0]);
475 var current_marc = md.attr('value');
476 var sf_list = newsf.split('');
478 for (var i in sf_list) {
479 var re = '\\$' + sf_list[i];
480 if (current_marc.match(re)) continue;
481 current_marc += '$' + sf_list[i];
484 md.attr('value', current_marc);
489 var matchSF = dojo.query('*[name=matchSF]',clone).instantiate(dijit.form.TextBox, {
490 onChange : render_preview
493 var matchRE = dojo.query('*[name=matchRE]',clone).instantiate(dijit.form.TextBox, {
494 onChange : render_preview
497 var removeButton = dojo.query('*[name=removeButton]',clone).instantiate(dijit.form.Button, {
498 onClick : function() {
500 dojo.byId('ruleList').childNodes[kid_number],
507 dojo.place(clone,'ruleList');
510 function ruleset_to_record () {
511 var rec = new MARC.Record ({ delimiter : '$' });
514 dojo.query('#ruleList *[name=ruleTable]').filter( function (node) {
515 if (node.className.match(/hide_me/)) return false;
519 var rule_tag = new MARC.Field ({
524 var rule_txt = dijit.byNode(dojo.query('*[name=tagContainer] .dijit',tbl)[0]).attr('value');
525 rule_txt += dijit.byNode(dojo.query('*[name=sfContainer] .dijit',tbl)[0]).attr('value');
527 var reSF = dijit.byNode(dojo.query('*[name=matchSFContainer] .dijit',tbl)[0]).attr('value');
529 var reRE = dijit.byNode(dojo.query('*[name=matchREContainer] .dijit',tbl)[0]).attr('value');
530 rule_txt += '[' + reSF + '~' + reRE + ']';
533 var rtype = dijit.byNode(dojo.query('*[name=typeSelectContainer] .dijit',tbl)[0]).attr('value');
534 rule_tag.addSubfields( rtype, rule_txt )
535 rec.appendFields( rule_tag );
537 if (rtype == 'a' || rtype == 'r') {
541 marcbreaker : dijit.byNode(dojo.query('*[name=marcDataContainer] .dijit',tbl)[0]).attr('value')
553 <body style="margin:10px;" class='tundra'>
555 <div dojoType="dijit.form.Form" id="myForm" jsId="myForm" encType="multipart/form-data" action="" method="POST">
556 <script type='dojo/method' event='onSubmit'>
557 var rec = ruleset_to_record();
559 if (rec.subfield('905','r') == '') { // no-op to force replace mode
565 subfields : [['r','901c']]
570 dojo.byId('template_value').value = rec.toXmlString();
574 <input type='hidden' id='template_value' name='template'/>
576 <label for='inputTypeSelect'>Record source:</label>
577 <select name='recordSource' dojoType='dijit.form.FilteringSelect'>
578 <script type='dojo/method' event='onChange' args="val">
581 dojo.removeClass('bucketListContainer','hide_me');
582 dojo.addClass('csvContainer','hide_me');
583 dojo.addClass('recordContainer','hide_me');
586 dojo.addClass('bucketListContainer','hide_me');
587 dojo.removeClass('csvContainer','hide_me');
588 dojo.addClass('recordContainer','hide_me');
591 dojo.addClass('bucketListContainer','hide_me');
592 dojo.addClass('csvContainer','hide_me');
593 dojo.removeClass('recordContainer','hide_me');
597 <script type='dojo/method' event='postCreate'>
598 if (cgi.param('recordSource')) {
599 this.attr('value',cgi.param('recordSource'));
600 this.onChange(cgi.param('recordSource'));
603 <option value='b'>a Bucket</option>
604 <option value='c'>a CSV File</option>
605 <option value='r'>a specific record ID</option>
608 <table style='margin:10px; margin-bottom:20px;'>
611 <th>Merge template name (optional):</th>
612 <td><input id='bucketName' jsId='bucketName' type='text' dojoType='dijit.form.TextBox' name='bname' value=''/></td>
615 <tr class='' id='bucketListContainer'>
617 <div name='containerid' jsId='bucketList' dojoType='dijit.form.FilteringSelect' store='bucketStore' searchAttr='name' id='bucketList'>
618 <script type='dojo/method' event='postCreate'>
619 if (cgi.param('containerid')) this.attr('value',cgi.param('containerid'));
624 <tr class='hide_me' id='csvContainer'>
626 Column <input style='width:75px;' type='text' dojoType='dijit.form.NumberSpinner' name='idcolumn' value='0' constraints='{min:0,max:100,places:0}' /> of:
627 <input id='idfile' type="file" name="idfile"/>
630 Columns are numbered starting at 0. For instance, when looking at a CSV file in Excel, the column labeled A is the same as column 0, and the column labeled B is the same as column 1.
633 <tr class='hide_me' id='recordContainer'>
634 <td>Record ID: <input dojoType='dijit.form.TextBox' name='recid' style='width:75px;' type='text' value=''/></td>
638 <button type="submit" dojoType='dijit.form.Button'>GO!</button> (After setting up your template below.)
643 </div> <!-- end of the form -->
646 <table style='width: 100%'>
648 <td style='width: 50%'><div id='ruleList'></div></td>
649 <td valign='top'>Update Template Preview:<br/><pre id='marcPreview'></pre></td>
653 <button dojoType='dijit.form.Button'>Add Merge Rule
654 <script type='dojo/connect' event='onClick'>render_from_template()</script>
655 <script type='dojo/method' event='postCreate'>render_from_template()</script>
658 <div class='hide_me' id='ruleTemplate'>
659 <div name='ruleTable'>
660 <table class='ruleTable'>
663 <th style="text-align:center;">Rule Setup</th>
664 <th style="text-align:center;">Data</th>
665 <th style="text-align:center;">Help</th>
668 <th>Action (Rule Type)</th>
669 <td name='typeSelectContainer'>
670 <select name='typeSelect'>
671 <option value='r'>Replace</option>
672 <option value='a'>Add</option>
673 <option value='d'>Delete</option>
676 <td>How to change the existing records</td>
680 <td name='tagContainer'><input style='with: 2em;' name='tag' type='text'></input</td>
681 <td>Three characters, no spaces, no indicators, etc. eg: 245</td>
684 <th>Subfields (optional)</th>
685 <td name='sfContainer'><input name='sf' type='text'/></td>
686 <td>No spaces, no delimiters, eg: abcnp</td>
690 <td name='marcDataContainer'><input name='marcData' type='text'/></td>
691 <td>MARC-Breaker formatted data with indicators and subfield delimiters, eg:<br/>245 04$aThe End</td>
694 <th colspan='3' style='padding-top: 20px; text-align: center;'>Advanced Matching Restriction (Optional)</th>
698 <td name='matchSFContainer'><input style='with: 2em;' name='matchSF' type='text'></input</td>
699 <td>A single subfield code, no delimiters, eg: a</td>
701 <th>Regular Expression</th>
702 <td name='matchREContainer'><input name='matchRE' type='text'/></td>
703 <td>See <a href="http://perldoc.perl.org/perlre.html#Regular-Expressions" target="_blank">the Perl documentation</a>
704 for an explanation of Regular Expressions.
708 <td colspan='3' style='padding-top: 20px; text-align: center;'>
709 <button name='removeButton'>Remove this Template Rule</button>
722 return Apache2::Const::OK;