From 90c205f7cbc3f244b47f82d1c9529173cfa837e7 Mon Sep 17 00:00:00 2001 From: Bill Erickson Date: Thu, 21 May 2020 12:06:38 -0400 Subject: [PATCH] LP1880726 MARC Batch edit Angular port Angular port of the MARC Batch Edit interface. Under the covers, each bib record is now modified within its own transaction to avoid long-running transactions that can potentially lock database rows needed by other processes. Signed-off-by: Bill Erickson Signed-off-by: Mike Rylander --- .../cat/marcbatch/marcbatch.component.html | 187 ++++++++++++ .../cat/marcbatch/marcbatch.component.ts | 269 ++++++++++++++++++ .../staff/cat/marcbatch/marcbatch.module.ts | 23 ++ .../app/staff/cat/marcbatch/routing.module.ts | 17 ++ .../eg2/src/app/staff/cat/routing.module.ts | 4 + .../src/eg2/src/app/staff/nav.component.html | 2 +- .../app/staff/share/marc-edit/marcrecord.ts | 12 +- .../perlmods/lib/OpenILS/Application/Cat.pm | 24 +- .../lib/OpenILS/WWW/TemplateBatchBibUpdate.pm | 33 ++- Open-ILS/src/sql/Pg/950.data.seed-values.sql | 1 - Open-ILS/src/templates/staff/navbar.tt2 | 2 +- .../Cataloging/marcbatch-ang-port.adoc | 3 + 12 files changed, 564 insertions(+), 13 deletions(-) create mode 100644 Open-ILS/src/eg2/src/app/staff/cat/marcbatch/marcbatch.component.html create mode 100644 Open-ILS/src/eg2/src/app/staff/cat/marcbatch/marcbatch.component.ts create mode 100644 Open-ILS/src/eg2/src/app/staff/cat/marcbatch/marcbatch.module.ts create mode 100644 Open-ILS/src/eg2/src/app/staff/cat/marcbatch/routing.module.ts create mode 100644 docs/RELEASE_NOTES_NEXT/Cataloging/marcbatch-ang-port.adoc diff --git a/Open-ILS/src/eg2/src/app/staff/cat/marcbatch/marcbatch.component.html b/Open-ILS/src/eg2/src/app/staff/cat/marcbatch/marcbatch.component.html new file mode 100644 index 0000000000..d3cbb1306d --- /dev/null +++ b/Open-ILS/src/eg2/src/app/staff/cat/marcbatch/marcbatch.component.html @@ -0,0 +1,187 @@ + + +
+
+ +
+
+
Rule Setup
+
Data
+
Help
+
+
+
Action (Rule Type)
+
+ +
+
How to change the existing record.
+
+
+
MARC Tag
+
+ +
+
+ Three characters, no spaces, no indicators, etc. eg: 245 +
+
+
+
Subfields (optional)
+
+ +
+
No spaces, no delimiters, eg: abcnp
+
+
+
MARC Data
+
+ +
+
+ MARC-Breaker formatted data with indicators and subfield delimiters, + eg: 245 04$aThe End +
+
+
+
+
+ Advanced Matching Restriction (Optional) +
+
+
+
+
Subfield
+
+ +
+
+ A single subfield code, no delimiters, eg: a +
+
+
+
Expression
+
+ +
+
+ See the + + Perl documentation + for an explanation of Regular Expressions. +
+
+
+
+ +
+
+ +
+
+ +
+
+
+
+
+
+
Merge Template Preview
+
+ +
+
+
+
+
Record Source:
+
+ +
+
+
+ +
Bucket named:
+
+ + +
+ + +
Record ID:
+
+ +
+ + +
+
+
Column:
+
+ + of +
+
+ +
+
+
+
+ Columns are numbered starting at 0. For instance, when looking + at a CSV file in Excel, the column labeled A is the same as + column 0, and the column labeled B is the same as column 1. +
+
+
+ +
+
+
+ +
+
+
+
+ + +
+
+
+
+
Processing Complete
+
+
Success count:
+
{{this.numSucceeded}}
+
+
+
Failed count:
+
{{this.numFailed}}
+
+
+
+
+
diff --git a/Open-ILS/src/eg2/src/app/staff/cat/marcbatch/marcbatch.component.ts b/Open-ILS/src/eg2/src/app/staff/cat/marcbatch/marcbatch.component.ts new file mode 100644 index 0000000000..84c7f354dd --- /dev/null +++ b/Open-ILS/src/eg2/src/app/staff/cat/marcbatch/marcbatch.component.ts @@ -0,0 +1,269 @@ +import {Component, OnInit, ViewChild, Renderer2} from '@angular/core'; +import {Router, ActivatedRoute, ParamMap} from '@angular/router'; +import {HttpClient} from '@angular/common/http'; +import {tap} from 'rxjs/operators'; +import {NetService} from '@eg/core/net.service'; +import {AuthService} from '@eg/core/auth.service'; +import {PcrudService} from '@eg/core/pcrud.service'; +import {ComboboxEntry} from '@eg/share/combobox/combobox.component'; +import {MarcRecord, MarcField} from '@eg/staff/share/marc-edit/marcrecord'; +import {AnonCacheService} from '@eg/share/util/anon-cache.service'; +import {ServerStoreService} from '@eg/core/server-store.service'; + +const SESSION_POLL_INTERVAL = 2; // seconds +const MERGE_TEMPLATE_PATH = '/opac/extras/merge_template'; + +interface TemplateRule { + ruleType: 'r' | 'a' | 'd'; + marcTag?: string; + marcSubfields?: string; + marcData?: string; + advSubfield?: string; + advRegex?: string; +} + +@Component({ + templateUrl: 'marcbatch.component.html' +}) +export class MarcBatchComponent implements OnInit { + + session: string; + source: 'b' | 'c' | 'r' = 'b'; + buckets: ComboboxEntry[]; + bucket: number; + recordId: number; + csvColumn = 0; + csvFile: File; + templateRules: TemplateRule[] = []; + record: MarcRecord; + + processing = false; + progressMax: number = null; + progressValue: number = null; + numSucceeded = 0; + numFailed = 0; + + constructor( + private router: Router, + private route: ActivatedRoute, + private http: HttpClient, + private renderer: Renderer2, + private net: NetService, + private pcrud: PcrudService, + private auth: AuthService, + private store: ServerStoreService, + private cache: AnonCacheService + ) {} + + ngOnInit() { + this.load(); + } + + load() { + this.addRule(); + this.getBuckets(); + } + + rulesetToRecord(resetRuleData?: boolean) { + this.record = new MarcRecord(); + + this.templateRules.forEach(rule => { + + if (!rule.marcTag) { return; } + + let ruleText = rule.marcTag + (rule.marcSubfields || ''); + if (rule.advSubfield) { + ruleText += + `[${rule.advSubfield || ''} ~ ${rule.advRegex || ''}]`; + } + + // Merge behavior is encoded in the 905 field. + const ruleTag = this.record.newField({ + tag: '905', + ind1: ' ', + ind2: ' ', + subfields: [[rule.ruleType, ruleText, 0]] + }); + + this.record.insertOrderedFields(ruleTag); + + if (rule.ruleType === 'd') { + rule.marcData = ''; + return; + } + + const dataRec = new MarcRecord(); + if (resetRuleData || !rule.marcData) { + + // Build a new value for the 'MARC Data' field based on + // changes to the selected tag or subfields. + + const subfields = rule.marcSubfields ? + rule.marcSubfields.split('').map((sf, idx) => [sf, '', idx]) + : []; + + dataRec.appendFields( + dataRec.newField({ + tag: rule.marcTag, + ind1: ' ', + ind2: ' ', + subfields: subfields + }) + ); + + console.log(dataRec.toBreaker()); + rule.marcData = dataRec.toBreaker().split(/\n/)[1]; + + } else { + + // Absorb the breaker data already in the 'MARC Data' field + // so it can be added to the template record in progress. + + dataRec.breakerText = rule.marcData; + dataRec.absorbBreakerChanges(); + } + + this.record.appendFields(dataRec.fields[0]); + }); + } + + breakerRows(): number { + if (this.record) { + const breaker = this.record.toBreaker(); + if (breaker) { + return breaker.split(/\n/).length + 1; + } + } + return 3; + } + + breaker(): string { + return this.record ? this.record.toBreaker() : ''; + } + + addRule() { + this.templateRules.push({ruleType: 'r'}); + } + + removeRule(idx: number) { + this.templateRules.splice(idx, 1); + } + + getBuckets(): Promise { + if (this.buckets) { return Promise.resolve(); } + + return this.net.request( + 'open-ils.actor', + 'open-ils.actor.container.retrieve_by_class', + this.auth.token(), this.auth.user().id(), + 'biblio', ['staff_client', 'vandelay_queue'] + + ).pipe(tap(buckets => { + this.buckets = buckets + .sort((b1, b2) => b1.name() < b2.name() ? -1 : 1) + .map(b => ({id: b.id(), label: b.name()})); + + })).toPromise(); + } + + bucketChanged(entry: ComboboxEntry) { + this.bucket = entry ? entry.id : null; + } + + fileSelected($event) { + this.csvFile = $event.target.files[0]; + } + + disableSave(): boolean { + if (!this.record || !this.source || this.processing) { + return true; + } + + if (this.source === 'b') { + return !this.bucket; + + } else if (this.source === 'c') { + return (!this.csvColumn || !this.csvFile); + + } else if (this.source === 'r') { + return !this.recordId; + } + } + + process() { + this.processing = true; + this.progressValue = null; + this.progressMax = null; + this.numSucceeded = 0; + this.numFailed = 0; + this.setReplaceMode(); + this.postForm().then(_ => this.pollProgress()); + } + + setReplaceMode() { + if (this.record.subfield('905', 'r').length === 0) { + // Force replace mode w/ no-op replace rule. + this.record.appendFields( + this.record.newField({ + tag : '905', + ind1 : ' ', + ind2 : ' ', + subfields : [['r', '901c']] + }) + ); + } + } + + postForm(): Promise { + + const formData: FormData = new FormData(); + formData.append('ses', this.auth.token()); + formData.append('skipui', '1'); + formData.append('template', this.record.toXml()); + formData.append('recordSource', this.source); + formData.append('xactPerRecord', '1'); + + if (this.source === 'b') { + formData.append('containerid', this.bucket + ''); + + } else if (this.source === 'c') { + formData.append('idcolumn', this.csvColumn + ''); + formData.append('idfile', this.csvFile, this.csvFile.name); + + } else if (this.source === 'r') { + formData.append('recid', this.recordId + ''); + } + + return this.http.post( + MERGE_TEMPLATE_PATH, formData, {responseType: 'text'}) + .pipe(tap(cacheKey => this.session = cacheKey)) + .toPromise(); + } + + pollProgress(): Promise { + console.debug('Polling session ', this.session); + + return this.cache.getItem(this.session, 'batch_edit_progress') + .then(progress => { + // {"success":"t","complete":1,"failed":0,"succeeded":252} + + if (!progress) { + console.error('No batch edit session found for ', this.session); + return; + } + + this.progressValue = progress.succeeded; + this.progressMax = progress.total; + this.numSucceeded = progress.succeeded; + this.numFailed = progress.failed; + + if (progress.complete) { + this.processing = false; + return; + } + + setTimeout(() => this.pollProgress(), SESSION_POLL_INTERVAL * 1000); + }); + } +} + diff --git a/Open-ILS/src/eg2/src/app/staff/cat/marcbatch/marcbatch.module.ts b/Open-ILS/src/eg2/src/app/staff/cat/marcbatch/marcbatch.module.ts new file mode 100644 index 0000000000..bf81553919 --- /dev/null +++ b/Open-ILS/src/eg2/src/app/staff/cat/marcbatch/marcbatch.module.ts @@ -0,0 +1,23 @@ +import {NgModule} from '@angular/core'; +import {StaffCommonModule} from '@eg/staff/common.module'; +import {CommonWidgetsModule} from '@eg/share/common-widgets.module'; +import {MarcBatchRoutingModule} from './routing.module'; +import {MarcBatchComponent} from './marcbatch.component'; +import {HttpClientModule} from '@angular/common/http'; + +@NgModule({ + declarations: [ + MarcBatchComponent + ], + imports: [ + StaffCommonModule, + HttpClientModule, + CommonWidgetsModule, + MarcBatchRoutingModule + ], + providers: [ + ] +}) + +export class MarcBatchModule { +} diff --git a/Open-ILS/src/eg2/src/app/staff/cat/marcbatch/routing.module.ts b/Open-ILS/src/eg2/src/app/staff/cat/marcbatch/routing.module.ts new file mode 100644 index 0000000000..bb268e644d --- /dev/null +++ b/Open-ILS/src/eg2/src/app/staff/cat/marcbatch/routing.module.ts @@ -0,0 +1,17 @@ +import {NgModule} from '@angular/core'; +import {RouterModule, Routes} from '@angular/router'; +import {MarcBatchComponent} from './marcbatch.component'; + +const routes: Routes = [{ + path: '', + component: MarcBatchComponent +}]; + +@NgModule({ + imports: [RouterModule.forChild(routes)], + exports: [RouterModule], + providers: [] +}) + +export class MarcBatchRoutingModule {} + diff --git a/Open-ILS/src/eg2/src/app/staff/cat/routing.module.ts b/Open-ILS/src/eg2/src/app/staff/cat/routing.module.ts index d4b2770d8a..b3523789d9 100644 --- a/Open-ILS/src/eg2/src/app/staff/cat/routing.module.ts +++ b/Open-ILS/src/eg2/src/app/staff/cat/routing.module.ts @@ -9,6 +9,10 @@ const routes: Routes = [ path: 'authority', loadChildren: () => import('./authority/authority.module').then(m => m.AuthorityModule) + }, { + path: 'marcbatch', + loadChildren: () => + import('./marcbatch/marcbatch.module').then(m => m.MarcBatchModule) } ]; diff --git a/Open-ILS/src/eg2/src/app/staff/nav.component.html b/Open-ILS/src/eg2/src/app/staff/nav.component.html index a36ce00386..77253416cb 100644 --- a/Open-ILS/src/eg2/src/app/staff/nav.component.html +++ b/Open-ILS/src/eg2/src/app/staff/nav.component.html @@ -206,7 +206,7 @@ MARC Batch Import/Export - + MARC Batch Edit diff --git a/Open-ILS/src/eg2/src/app/staff/share/marc-edit/marcrecord.ts b/Open-ILS/src/eg2/src/app/staff/share/marc-edit/marcrecord.ts index d49f4ef568..493eaadcbe 100644 --- a/Open-ILS/src/eg2/src/app/staff/share/marc-edit/marcrecord.ts +++ b/Open-ILS/src/eg2/src/app/staff/share/marc-edit/marcrecord.ts @@ -60,7 +60,7 @@ export class MarcRecord { this.record.fields = f; } - constructor(xml: string) { + constructor(xml?: string) { this.record = new MARC21.Record({marcxml: xml, delimiter: DELIMITER}); this.breakerText = this.record.toBreaker(); this.fixedFieldChange = new EventEmitter(); @@ -113,6 +113,11 @@ export class MarcRecord { return this.record.field(spec, wantArray); } + appendFields(...newFields: MarcField[]) { + this.record.appendFields.apply(this.record, newFields); + this.stampFieldIds(); + } + insertFieldsBefore(field: MarcField, ...newFields: MarcField[]) { this.record.insertFieldsBefore.apply( this.record, [field].concat(newFields)); @@ -186,5 +191,10 @@ export class MarcRecord { subfields.forEach(sf => root.push([].concat(sf))); return root; } + + // Returns a list of values for the tag + subfield combo + subfield(tag: string, subfield: string): string { + return this.record.subfield(tag, subfield); + } } diff --git a/Open-ILS/src/perlmods/lib/OpenILS/Application/Cat.pm b/Open-ILS/src/perlmods/lib/OpenILS/Application/Cat.pm index 4cfcfbd573..36376a0948 100644 --- a/Open-ILS/src/perlmods/lib/OpenILS/Application/Cat.pm +++ b/Open-ILS/src/perlmods/lib/OpenILS/Application/Cat.pm @@ -240,12 +240,17 @@ __PACKAGE__->register_method( @param auth The authtoken @param container The container, um, containing the records to be updated by the template @param template The overlay template, or nothing and the method will look for a negative bib id in the container + @param options Hash of options; currently supports: + xact_per_record: Apply updates to each bib record within its own transaction. @return Cache key to check for status of the container overlay # ); sub template_overlay_container { - my($self, $conn, $auth, $container, $template) = @_; + my($self, $conn, $auth, $container, $template, $options) = @_; + $options ||= {}; + my $xact_per_rec = $options->{xact_per_record}; + my $e = new_editor(authtoken=>$auth, xact=>1); return $e->die_event unless $e->checkauth; @@ -265,14 +270,20 @@ sub template_overlay_container { $template = $e->retrieve_biblio_record_entry( $titem->target_biblio_record_entry )->marc; } + my $num_total = scalar(@$items); my $num_failed = 0; my $num_succeeded = 0; $conn->respond_complete( - $actor->request('open-ils.actor.anon_cache.set_value', $auth, batch_edit_progress => {})->gather(1) + $actor->request('open-ils.actor.anon_cache.set_value', $auth, + batch_edit_progress => {total => $num_total})->gather(1) ) if ($actor); + # read-only up to here. + $e->rollback if $xact_per_rec; + for my $item ( @$items ) { + $e->xact_begin if $xact_per_rec; my $rec = $e->retrieve_biblio_record_entry($item->target_biblio_record_entry); next unless $rec; @@ -294,6 +305,7 @@ sub template_overlay_container { $actor->request( 'open-ils.actor.anon_cache.set_value', $auth, batch_edit_progress => { + total => $num_total, succeeded => $num_succeeded, failed => $num_failed }, @@ -311,6 +323,7 @@ sub template_overlay_container { batch_edit_progress => { complete => 1, success => 'f', + total => $num_total, succeeded => $num_succeeded, failed => $num_failed, } @@ -321,19 +334,23 @@ sub template_overlay_container { } } } + $e->xact_commit if $xact_per_rec; } if ($titem && !$num_failed) { + $e->xact_begin if $xact_per_rec; return $e->die_event unless ($e->delete_container_biblio_record_entry_bucket_item($titem)); + $e->xact_commit if $xact_per_rec; } - if ($e->commit) { + if ($xact_per_rec || $e->commit) { if ($actor) { $actor->request( 'open-ils.actor.anon_cache.set_value', $auth, batch_edit_progress => { complete => 1, success => 't', + total => $num_total, succeeded => $num_succeeded, failed => $num_failed, } @@ -348,6 +365,7 @@ sub template_overlay_container { batch_edit_progress => { complete => 1, success => 'f', + total => $num_total, succeeded => $num_succeeded, failed => $num_failed, } diff --git a/Open-ILS/src/perlmods/lib/OpenILS/WWW/TemplateBatchBibUpdate.pm b/Open-ILS/src/perlmods/lib/OpenILS/WWW/TemplateBatchBibUpdate.pm index 27a03e1107..ae91b36a94 100644 --- a/Open-ILS/src/perlmods/lib/OpenILS/WWW/TemplateBatchBibUpdate.pm +++ b/Open-ILS/src/perlmods/lib/OpenILS/WWW/TemplateBatchBibUpdate.pm @@ -4,7 +4,7 @@ use warnings; use bytes; use Apache2::Log; -use Apache2::Const -compile => qw(OK REDIRECT DECLINED NOT_FOUND :log); +use Apache2::Const -compile => qw(OK REDIRECT DECLINED NOT_FOUND HTTP_BAD_REQUEST :log); use APR::Const -compile => qw(:error SUCCESS); use APR::Table; @@ -56,14 +56,21 @@ sub handler { my $cgi = new CGI; my $authid = $cgi->cookie('ses') || $cgi->param('ses'); + + # Avoid sending the HTML to the caller. Final response will + # will just be the cache key or HTTP_BAD_REQUEST on error. + my $skipui = $cgi->param('skipui'); + my $usr = verify_login($authid); - return show_template($r) unless ($usr); + return show_template($r, $skipui) unless ($usr); my $template = $cgi->param('template'); - return show_template($r) unless ($template); + return show_template($r, $skipui) unless ($template); my $rsource = $cgi->param('recordSource'); + my $xact_per = $cgi->param('xactPerRecord'); + # find some IDs ... my @records; @@ -118,7 +125,7 @@ sub handler { unless (@records) { $e->request('open-ils.cstore.transaction.rollback')->gather(1); $e->disconnect; - return show_template($r); + return show_template($r, $skipui); } # we have a template and some record ids, so... @@ -173,10 +180,12 @@ sub handler { # fire the background bucket processor my $cache_key = OpenSRF::AppSession ->create('open-ils.cat') - ->request('open-ils.cat.container.template_overlay.background', $authid, $bucket->id) + ->request('open-ils.cat.container.template_overlay.background', + $authid, $bucket->id, undef, {xact_per_record => $xact_per}) ->gather(1); - return show_processing_template($r, $bucket->id, \@records, $cache_key); + return show_processing_template( + $r, $bucket->id, \@records, $cache_key, $skipui); } sub verify_login { @@ -201,6 +210,13 @@ sub show_processing_template { my $bid = shift; my $recs = shift; my $cache_key = shift; + my $skipui = shift; + + if ($skipui) { + $r->content_type('text/plain'); + $r->print($cache_key); + return Apache2::Const::OK; + } my $rec_string = @$recs; @@ -366,6 +382,11 @@ HTML sub show_template { my $r = shift; + my $skipui = shift; + + # Makes no sense to call the API in such a way that the caller + # is returned the UI code if skipui is set. + return Apache2::Const::HTTP_BAD_REQUEST if $skipui; $r->content_type('text/html'); $r->print(<<'HTML'); diff --git a/Open-ILS/src/sql/Pg/950.data.seed-values.sql b/Open-ILS/src/sql/Pg/950.data.seed-values.sql index 18564980f4..bf1ad50675 100644 --- a/Open-ILS/src/sql/Pg/950.data.seed-values.sql +++ b/Open-ILS/src/sql/Pg/950.data.seed-values.sql @@ -1,4 +1,3 @@ ---002.schema.config.sql: INSERT INTO config.bib_source (id, quality, source, transcendant, can_have_copies) VALUES (1, 90, oils_i18n_gettext(1, 'oclc', 'cbs', 'source'), FALSE, TRUE); INSERT INTO config.bib_source (id, quality, source, transcendant, can_have_copies) VALUES diff --git a/Open-ILS/src/templates/staff/navbar.tt2 b/Open-ILS/src/templates/staff/navbar.tt2 index 48b808f414..ac25e566a6 100644 --- a/Open-ILS/src/templates/staff/navbar.tt2 +++ b/Open-ILS/src/templates/staff/navbar.tt2 @@ -321,7 +321,7 @@
  • - + [% l('MARC Batch Edit') %] diff --git a/docs/RELEASE_NOTES_NEXT/Cataloging/marcbatch-ang-port.adoc b/docs/RELEASE_NOTES_NEXT/Cataloging/marcbatch-ang-port.adoc new file mode 100644 index 0000000000..b97b578495 --- /dev/null +++ b/docs/RELEASE_NOTES_NEXT/Cataloging/marcbatch-ang-port.adoc @@ -0,0 +1,3 @@ +MARC Batch Edit UI Angular Port +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +The MARC Batch Edit interface has been ported to Angular. -- 2.43.2