1 package OpenILS::WWW::TemplateBatchBibUpdate;
7 use Apache2::Const -compile => qw(OK REDIRECT DECLINED NOT_FOUND HTTP_BAD_REQUEST :log);
8 use APR::Const -compile => qw(:error SUCCESS);
11 use Apache2::RequestRec ();
12 use Apache2::RequestIO ();
13 use Apache2::RequestUtil;
18 use OpenSRF::EX qw(:try);
19 use OpenILS::Utils::DateTime qw/:datetime/;
20 use OpenSRF::Utils::Cache;
22 use OpenSRF::AppSession;
27 use Unicode::Normalize;
28 use OpenILS::Utils::Fieldmapper;
29 use OpenSRF::Utils::Logger qw/$logger/;
32 use MARC::File::XML ( BinaryEncoding => 'UTF-8' );
34 use UNIVERSAL::require;
36 our @formats = qw/USMARC UNIMARC XML BRE/;
38 # set the bootstrap config and template include directory when
39 # this module is loaded
49 OpenSRF::System->bootstrap_client( config_file => $bootstrap );
50 Fieldmapper->import(IDL => OpenSRF::Utils::SettingsClient->new->config_value("IDL"));
51 return Apache2::Const::OK;
58 my $authid = $cgi->cookie('ses') || $cgi->param('ses');
60 # Avoid sending the HTML to the caller. Final response will
61 # will just be the cache key or HTTP_BAD_REQUEST on error.
62 my $skipui = $cgi->param('skipui');
64 my $usr = verify_login($authid);
65 return show_template($r, $skipui) unless ($usr);
67 my $template = $cgi->param('template');
68 return show_template($r, $skipui) unless ($template);
71 my $rsource = $cgi->param('recordSource');
72 my $xact_per = $cgi->param('xactPerRecord');
77 if ($rsource eq 'r') {
78 @records = map { $_ ? ($_) : () } $cgi->param('recid');
81 if ($rsource eq 'c') { # try for a file
82 my $file = $cgi->param('idfile');
84 my $col = $cgi->param('idcolumn') || 0;
85 my $csv = new Text::CSV;
89 my @data = $csv->fields;
98 my $e = OpenSRF::AppSession->connect('open-ils.cstore');
99 $e->request('open-ils.cstore.transaction.begin')->gather(1);
100 $e->request('open-ils.cstore.set_audit_info', $authid, $usr->id, $usr->wsid)->gather(1);
102 # still no records ...
103 my $container = $cgi->param('containerid');
104 if ($rsource eq 'b') {
106 my $bucket = $e->request(
107 'open-ils.cstore.direct.container.biblio_record_entry_bucket.retrieve',
111 $e->request('open-ils.cstore.transaction.rollback')->gather(1);
113 $r->log->error("No such bucket $container");
114 $logger->error("No such bucket $container");
115 return Apache2::Const::NOT_FOUND;
117 my $recs = $e->request(
118 'open-ils.cstore.direct.container.biblio_record_entry_bucket_item.search.atomic',
119 { bucket => $container }
121 @records = map { ($_->target_biblio_record_entry) } @$recs;
126 $e->request('open-ils.cstore.transaction.rollback')->gather(1);
128 return show_template($r, $skipui);
131 # we have a template and some record ids, so...
133 # insert the template record
134 my $min_id = $e->request(
135 'open-ils.cstore.json_query',
136 { select => { bre => [{ column => 'id', transform => 'min', aggregate => 1}] }, from => 'bre' }
137 )->gather(1)->{id} - 1;
139 warn "new template bib id = $min_id\n";
141 my $tmpl_rec = Fieldmapper::biblio::record_entry->new;
142 $tmpl_rec->id($min_id);
143 $tmpl_rec->deleted('t');
144 $tmpl_rec->active('f');
145 $tmpl_rec->marc($template);
146 $tmpl_rec->creator($usr->id);
147 $tmpl_rec->editor($usr->id);
149 warn "about to create bib $min_id\n";
150 $e->request('open-ils.cstore.direct.biblio.record_entry.create', $tmpl_rec )->gather(1);
152 # create the new container for the records and the template
153 my $bucket = Fieldmapper::container::biblio_record_entry_bucket->new;
154 $bucket->owner($usr->id);
155 $bucket->btype('template_merge');
157 my $bname = $cgi->param('bname') || 'Temporary Merge Bucket '. localtime() . ' ' . $usr->id;
158 $bucket->name($bname);
160 $bucket = $e->request('open-ils.cstore.direct.container.biblio_record_entry_bucket.create', $bucket )->gather(1);
162 # create items in the bucket
163 my $item = Fieldmapper::container::biblio_record_entry_bucket_item->new;
164 $item->bucket($bucket->id);
165 $item->target_biblio_record_entry($min_id);
167 $e->request('open-ils.cstore.direct.container.biblio_record_entry_bucket_item.create', $item )->gather(1);
170 for my $r (@records) {
172 $item->target_biblio_record_entry($r);
173 $e->request('open-ils.cstore.direct.container.biblio_record_entry_bucket_item.create', $item )->gather(1);
177 $e->request('open-ils.cstore.transaction.commit')->gather(1);
180 # fire the background bucket processor
181 my $cache_key = OpenSRF::AppSession
182 ->create('open-ils.cat')
183 ->request('open-ils.cat.container.template_overlay.background',
184 $authid, $bucket->id, undef, {xact_per_record => $xact_per})
187 return show_processing_template(
188 $r, $bucket->id, \@records, $cache_key, $skipui);
192 my $auth_token = shift;
193 return undef unless $auth_token;
195 my $user = OpenSRF::AppSession
196 ->create("open-ils.auth")
197 ->request( "open-ils.auth.session.retrieve", $auth_token )
200 if (ref($user) eq 'HASH' && $user->{ilsevent} == 1001) {
204 return $user if ref($user);
208 sub show_processing_template {
212 my $cache_key = shift;
216 $r->content_type('text/plain');
217 $r->print($cache_key);
218 return Apache2::Const::OK;
221 my $rec_string = @$recs;
223 $r->content_type('text/html');
225 <html xmlns="http://www.w3.org/1999/xhtml">
228 <title>Merging records...</title>
229 <style type="text/css">
230 \@import '/js/dojo/dojo/resources/dojo.css';
231 \@import '/js/dojo/dijit/themes/tundra/tundra.css';
232 .hide_me { display: none; visibility: hidden; }
233 th { font-weight: bold; }
236 <script type="text/javascript">
240 AutoIDL: ['aou','aout','pgt','au','cbreb']
244 <script src='/js/dojo/dojo/dojo.js'></script>
245 <!-- <script src="/js/dojo/dojo/openils_dojo.js"></script> -->
247 <script type="text/javascript">
249 dojo.require('fieldmapper.AutoIDL');
250 dojo.require('fieldmapper.dojoData');
251 dojo.require('openils.User');
252 dojo.require('openils.XUL');
253 dojo.require('dojo.cookie');
254 dojo.require('openils.CGI');
255 dojo.require('openils.widget.ProgressDialog');
257 var cgi = new openils.CGI();
258 var authtoken = dojo.cookie('ses') || cgi.param('ses');
259 if (!authtoken && openils.XUL.isXUL()) {
260 var stash = openils.XUL.getStash();
261 authtoken = stash.session.key;
263 var u = new openils.User({ authtoken: authtoken });
265 dojo.addOnLoad(function () {
267 progress_dialog.update({maximum: $rec_string});
268 progress_dialog.attr("title", "MARC Batch Editor Progress......");
269 progress_dialog.show();
272 interval = setInterval( function() {
273 fieldmapper.standardRequest(
274 ['open-ils.actor','open-ils.actor.anon_cache.get_value'],
276 params: [ u.authtoken, 'batch_edit_progress' ],
277 onerror : function (r) { progress_dialog.hide(); },
278 onresponse : function (r) {
279 var counter = { success : 0, fail : 0, total : 0 };
280 if (x = openils.Util.readResponse(r)) {
281 counter.success = x.succeeded;
282 counter.fail = x.failed;
283 counter.total = counter.success + counter.fail;
285 clearInterval(interval);
286 progress_dialog.hide();
287 if (x.success == 't') dojo.byId('complete_msg').innerHTML = 'Overlay completed successfully';
288 else dojo.byId('complete_msg').innerHTML = 'Overlay did not complet successfully';
292 // update the progress dialog
293 progress_dialog.update({progress:counter.total});
294 dojo.byId('success_count').innerHTML = counter.success;
295 dojo.byId('fail_count').innerHTML = counter.fail;
296 dojo.byId('total_count').innerHTML = counter.total;
309 border: 1px solid black;
310 border-collapse: collapse;
316 table tr:nth-child(even) {
317 background-color: #eee;
319 table tr:nth-child(odd) {
320 background-color:#fff;
323 background-color: black;
336 text-decoration: underline;
342 <body style="margin:10px;font-size: 130%" class='tundra'>
343 <div class="hide_me"><div dojoType="openils.widget.ProgressDialog" jsId="progress_dialog"></div></div>
345 <h1>MARC Batch Editor Status</h1>
350 <th>Record Count</th>
354 <td id='success_count'></td>
358 <td id='fail_count'></td>
361 <td>Total Processed</td>
362 <td id='total_count'></td>
368 <td>Total To Process</td>
373 <div id='complete_msg'></div>
379 return Apache2::Const::OK;
387 # Makes no sense to call the API in such a way that the caller
388 # is returned the UI code if skipui is set.
389 return Apache2::Const::HTTP_BAD_REQUEST if $skipui;
391 $r->content_type('text/html');
393 <html xmlns="http://www.w3.org/1999/xhtml">
396 <title>Merge Template Builder</title>
397 <style type="text/css">
398 @import '/js/dojo/dojo/resources/dojo.css';
399 @import '/js/dojo/dijit/themes/tundra/tundra.css';
400 .hide_me { display: none; visibility: hidden; }
401 table.ruleTable th { padding: 5px; border-collapse: collapse; border-bottom: solid 1px gray; font-weight: bold; }
402 table.ruleTable td { padding: 5px; border-collapse: collapse; border-bottom: solid 1px gray; }
405 <script type="text/javascript">
409 AutoIDL: ['aou','aout','pgt','au','cbreb']
413 <script src='/js/dojo/dojo/dojo.js'></script>
414 <!-- <script src="/js/dojo/dojo/openils_dojo.js"></script> -->
416 <script type="text/javascript">
418 dojo.require('dojo.data.ItemFileReadStore');
419 dojo.require('dijit.form.Form');
420 dojo.require('dijit.form.NumberSpinner');
421 dojo.require('dijit.form.FilteringSelect');
422 dojo.require('dijit.form.TextBox');
423 dojo.require('dijit.form.Textarea');
424 dojo.require('dijit.form.Button');
425 dojo.require('MARC.Batch');
426 dojo.require('fieldmapper.AutoIDL');
427 dojo.require('fieldmapper.dojoData');
428 dojo.require('openils.User');
429 dojo.require('openils.CGI');
430 dojo.require('openils.XUL');
431 dojo.require('dojo.cookie');
433 var cgi = new openils.CGI();
434 var authtoken = dojo.cookie('ses') || cgi.param('ses');
435 if (!authtoken && openils.XUL.isXUL()) {
436 var stash = openils.XUL.getStash();
437 authtoken = stash.session.key;
439 var u = new openils.User({ authtoken: authtoken });
441 var bucketStore = new dojo.data.ItemFileReadStore(
442 { data : cbreb.toStoreData(
443 fieldmapper.standardRequest(
444 ['open-ils.actor','open-ils.actor.container.retrieve_by_class.authoritative'],
445 [u.authtoken, u.user.id(), 'biblio', ['staff_client','vandelay_queue']]
451 function render_preview () {
452 var rec = ruleset_to_record();
453 dojo.byId('marcPreview').innerHTML = rec.toBreaker();
456 function render_from_template () {
457 var kid_number = dojo.byId('ruleList').childNodes.length;
458 var clone = dojo.query('*[name=ruleTable]', dojo.byId('ruleTemplate'))[0].cloneNode(true);
460 var typeSelect = dojo.query('*[name=typeSelect]',clone).instantiate(dijit.form.FilteringSelect, {
461 onChange : function (val) {
465 dijit.byNode(dojo.query('*[name=marcDataContainer] .dijit',clone)[0]).attr('disabled',false);
468 dijit.byNode(dojo.query('*[name=marcDataContainer] .dijit',clone)[0]).attr('disabled',true);
474 var marcData = dojo.query('*[name=marcData]',clone).instantiate(dijit.form.TextBox, {
475 onChange : render_preview
479 var tag = dojo.query('*[name=tag]',clone).instantiate(dijit.form.TextBox, {
480 onChange : function (newtag) {
481 var md = dijit.byNode(dojo.query('*[name=marcDataContainer] .dijit',clone)[0]);
482 var current_marc = md.attr('value');
484 if (newtag.length == 3) {
485 if (current_marc.length == 0) newtag += ' \\\\';
486 if (current_marc.substr(0,3) != newtag) current_marc = newtag + current_marc.substr(3);
488 md.attr('value', current_marc);
493 var sf = dojo.query('*[name=sf]',clone).instantiate(dijit.form.TextBox, {
494 onChange : function (newsf) {
495 var md = dijit.byNode(dojo.query('*[name=marcDataContainer] .dijit',clone)[0]);
496 var current_marc = md.attr('value');
497 var sf_list = newsf.split('');
499 for (var i in sf_list) {
500 var re = '\\$' + sf_list[i];
501 if (current_marc.match(re)) continue;
502 current_marc += '$' + sf_list[i];
505 md.attr('value', current_marc);
510 var matchSF = dojo.query('*[name=matchSF]',clone).instantiate(dijit.form.TextBox, {
511 onChange : render_preview
514 var matchRE = dojo.query('*[name=matchRE]',clone).instantiate(dijit.form.TextBox, {
515 onChange : render_preview
518 var removeButton = dojo.query('*[name=removeButton]',clone).instantiate(dijit.form.Button, {
519 onClick : function() {
521 dojo.byId('ruleList').childNodes[kid_number],
528 dojo.place(clone,'ruleList');
531 function ruleset_to_record () {
532 var rec = new MARC.Record ({ delimiter : '$' });
535 dojo.query('#ruleList *[name=ruleTable]').filter( function (node) {
536 if (node.className.match(/hide_me/)) return false;
540 var rule_tag = new MARC.Field ({
545 var rule_txt = dijit.byNode(dojo.query('*[name=tagContainer] .dijit',tbl)[0]).attr('value');
546 rule_txt += dijit.byNode(dojo.query('*[name=sfContainer] .dijit',tbl)[0]).attr('value');
548 var reSF = dijit.byNode(dojo.query('*[name=matchSFContainer] .dijit',tbl)[0]).attr('value');
550 var reRE = dijit.byNode(dojo.query('*[name=matchREContainer] .dijit',tbl)[0]).attr('value');
551 rule_txt += '[' + reSF + '~' + reRE + ']';
554 var rtype = dijit.byNode(dojo.query('*[name=typeSelectContainer] .dijit',tbl)[0]).attr('value');
555 rule_tag.addSubfields( rtype, rule_txt )
556 rec.appendFields( rule_tag );
558 if (rtype == 'a' || rtype == 'r') {
562 marcbreaker : dijit.byNode(dojo.query('*[name=marcDataContainer] .dijit',tbl)[0]).attr('value')
574 <body style="margin:10px;" class='tundra'>
576 <div dojoType="dijit.form.Form" id="myForm" jsId="myForm" encType="multipart/form-data" action="" method="POST">
577 <script type='dojo/method' event='onSubmit'>
578 var rec = ruleset_to_record();
580 if (rec.subfield('905','r') == '') { // no-op to force replace mode
586 subfields : [['r','901c']]
591 dojo.byId('template_value').value = rec.toXmlString();
595 <input type='hidden' id='template_value' name='template'/>
597 <label for='inputTypeSelect'>Record source:</label>
598 <select name='recordSource' dojoType='dijit.form.FilteringSelect'>
599 <script type='dojo/method' event='onChange' args="val">
602 dojo.removeClass('bucketListContainer','hide_me');
603 dojo.addClass('csvContainer','hide_me');
604 dojo.addClass('recordContainer','hide_me');
607 dojo.addClass('bucketListContainer','hide_me');
608 dojo.removeClass('csvContainer','hide_me');
609 dojo.addClass('recordContainer','hide_me');
612 dojo.addClass('bucketListContainer','hide_me');
613 dojo.addClass('csvContainer','hide_me');
614 dojo.removeClass('recordContainer','hide_me');
618 <script type='dojo/method' event='postCreate'>
619 if (cgi.param('recordSource')) {
620 this.attr('value',cgi.param('recordSource'));
621 this.onChange(cgi.param('recordSource'));
624 <option value='b'>a Bucket</option>
625 <option value='c'>a CSV File</option>
626 <option value='r'>a specific record ID</option>
629 <table style='margin:10px; margin-bottom:20px;'>
632 <th>Merge template name (optional):</th>
633 <td><input id='bucketName' jsId='bucketName' type='text' dojoType='dijit.form.TextBox' name='bname' value=''/></td>
636 <tr class='' id='bucketListContainer'>
638 <div name='containerid' jsId='bucketList' dojoType='dijit.form.FilteringSelect' store='bucketStore' searchAttr='name' id='bucketList'>
639 <script type='dojo/method' event='postCreate'>
640 if (cgi.param('containerid')) this.attr('value',cgi.param('containerid'));
645 <tr class='hide_me' id='csvContainer'>
647 Column <input style='width:75px;' type='text' dojoType='dijit.form.NumberSpinner' name='idcolumn' value='0' constraints='{min:0,max:100,places:0}' /> of:
648 <input id='idfile' type="file" name="idfile"/>
651 Columns are numbered starting at 0. For instance, when looking at a CSV file in Excel, the column labeled A is the same as column 0, and the column labeled B is the same as column 1.
654 <tr class='hide_me' id='recordContainer'>
655 <td>Record ID: <input dojoType='dijit.form.TextBox' name='recid' style='width:75px;' type='text' value=''/></td>
659 <button type="submit" dojoType='dijit.form.Button'>GO!</button> (After setting up your template below.)
664 </div> <!-- end of the form -->
667 <table style='width: 100%'>
669 <td style='width: 50%'><div id='ruleList'></div></td>
670 <td valign='top'>Update Template Preview:<br/><pre id='marcPreview'></pre></td>
674 <button dojoType='dijit.form.Button'>Add Merge Rule
675 <script type='dojo/connect' event='onClick'>render_from_template()</script>
676 <script type='dojo/method' event='postCreate'>render_from_template()</script>
679 <div class='hide_me' id='ruleTemplate'>
680 <div name='ruleTable'>
681 <table class='ruleTable'>
684 <th style="text-align:center;">Rule Setup</th>
685 <th style="text-align:center;">Data</th>
686 <th style="text-align:center;">Help</th>
689 <th>Action (Rule Type)</th>
690 <td name='typeSelectContainer'>
691 <select name='typeSelect'>
692 <option value='r'>Replace</option>
693 <option value='a'>Add</option>
694 <option value='d'>Delete</option>
697 <td>How to change the existing records</td>
701 <td name='tagContainer'><input style='with: 2em;' name='tag' type='text'></input</td>
702 <td>Three characters, no spaces, no indicators, etc. eg: 245</td>
705 <th>Subfields (optional)</th>
706 <td name='sfContainer'><input name='sf' type='text'/></td>
707 <td>No spaces, no delimiters, eg: abcnp</td>
711 <td name='marcDataContainer'><input name='marcData' type='text'/></td>
712 <td>MARC-Breaker formatted data with indicators and subfield delimiters, eg:<br/>245 04$aThe End</td>
715 <th colspan='3' style='padding-top: 20px; text-align: center;'>Advanced Matching Restriction (Optional)</th>
719 <td name='matchSFContainer'><input style='with: 2em;' name='matchSF' type='text'></input</td>
720 <td>A single subfield code, no delimiters, eg: a</td>
722 <th>Regular Expression</th>
723 <td name='matchREContainer'><input name='matchRE' type='text'/></td>
724 <td>See <a href="http://perldoc.perl.org/perlre.html#Regular-Expressions" target="_blank">the Perl documentation</a>
725 for an explanation of Regular Expressions.
729 <td colspan='3' style='padding-top: 20px; text-align: center;'>
730 <button name='removeButton'>Remove this Template Rule</button>
743 return Apache2::Const::OK;