Open-ILS/src/eg2/src/app/staff/cat/vandelay/import.component.ts

   1 import {Component, OnInit, AfterViewInit, Input,
   2     ViewChild, OnDestroy} from '@angular/core';
   3 import {Subject} from 'rxjs';
   4 import {tap} from 'rxjs/operators';
   5 import {IdlObject} from '@eg/core/idl.service';
   6 import {NetService} from '@eg/core/net.service';
   7 import {EventService} from '@eg/core/event.service';
   8 import {OrgService} from '@eg/core/org.service';
   9 import {AuthService} from '@eg/core/auth.service';
  10 import {ToastService} from '@eg/share/toast/toast.service';
  11 import {ComboboxComponent,
  12     ComboboxEntry} from '@eg/share/combobox/combobox.component';
  13 import {VandelayService, VandelayImportSelection,
  14   VANDELAY_UPLOAD_PATH} from './vandelay.service';
  15 import {HttpClient, HttpRequest, HttpEventType} from '@angular/common/http';
  16 import {HttpResponse, HttpErrorResponse} from '@angular/common/http';
  17 import {ProgressInlineComponent} from '@eg/share/dialog/progress-inline.component';
  18 import {AlertDialogComponent} from '@eg/share/dialog/alert.component';
  19 import {ServerStoreService} from '@eg/core/server-store.service';
  20
  21 const TEMPLATE_SETTING_NAME = 'eg.cat.vandelay.import.templates';
  22
  23 const TEMPLATE_ATTRS = [
  24     'recordType',
  25     'selectedBibSource',
  26     'selectedMatchSet',
  27     'mergeOnExact',
  28     'importNonMatching',
  29     'mergeOnBestMatch',
  30     'mergeOnSingleMatch',
  31     'autoOverlayAcqCopies',
  32     'selectedHoldingsProfile',
  33     'selectedMergeProfile',
  34     'selectedFallThruMergeProfile',
  35     'selectedTrashGroups',
  36     'minQualityRatio'
  37 ];
  38
  39 interface ImportOptions {
  40     session_key: string;
  41     overlay_map?: {[qrId: number]: /* breId */ number};
  42     import_no_match?: boolean;
  43     auto_overlay_exact?: boolean;
  44     auto_overlay_best_match?: boolean;
  45     auto_overlay_1match?: boolean;
  46     opp_acq_copy_overlay?: boolean;
  47     merge_profile?: any;
  48     fall_through_merge_profile?: any;
  49     strip_field_groups?: number[];
  50     match_quality_ratio: number;
  51     exit_early: boolean;
  52 }
  53
  54 @Component({
  55   templateUrl: 'import.component.html'
  56 })
  57 export class ImportComponent implements OnInit, AfterViewInit, OnDestroy {
  58
  59     recordType: string;
  60     selectedQueue: ComboboxEntry; // freetext enabled
  61
  62     // used for applying a default queue ID value when we have
  63     // a load-time queue before the queue combobox entries exist.
  64     startQueueId: number;
  65
  66     bibTrashGroups: IdlObject[];
  67     selectedTrashGroups: number[];
  68
  69     activeQueueId: number;
  70     selectedBucket: number;
  71     selectedBibSource: number;
  72     selectedMatchSet: number;
  73     selectedHoldingsProfile: number;
  74     selectedMergeProfile: number;
  75     selectedFallThruMergeProfile: number;
  76     selectedFile: File;
  77
  78     defaultMatchSet: string;
  79
  80     importNonMatching: boolean;
  81     mergeOnExact: boolean;
  82     mergeOnSingleMatch: boolean;
  83     mergeOnBestMatch: boolean;
  84     minQualityRatio: number;
  85     autoOverlayAcqCopies: boolean;
  86
  87     // True after the first upload, then remains true.
  88     showProgress: boolean;
  89
  90     // Upload in progress.
  91     isUploading: boolean;
  92
  93     // True only after successful upload
  94     uploadComplete: boolean;
  95
  96     // Upload / processsing session key
  97     // Generated by the server
  98     sessionKey: string;
  99
 100     // Optional enqueue/import tracker session name.
 101     sessionName: string;
 102
 103     selectedTemplate: string;
 104     formTemplates: {[name: string]: any};
 105     newTemplateName: string;
 106
 107     @ViewChild('fileSelector') private fileSelector;
 108     @ViewChild('uploadProgress')
 109         private uploadProgress: ProgressInlineComponent;
 110     @ViewChild('enqueueProgress')
 111         private enqueueProgress: ProgressInlineComponent;
 112     @ViewChild('importProgress')
 113         private importProgress: ProgressInlineComponent;
 114
 115     // Need these refs so values can be applied via external stimuli
 116     @ViewChild('formTemplateSelector')
 117         private formTemplateSelector: ComboboxComponent;
 118     @ViewChild('recordTypeSelector')
 119         private recordTypeSelector: ComboboxComponent;
 120     @ViewChild('bibSourceSelector')
 121         private bibSourceSelector: ComboboxComponent;
 122     @ViewChild('matchSetSelector')
 123         private matchSetSelector: ComboboxComponent;
 124     @ViewChild('holdingsProfileSelector')
 125         private holdingsProfileSelector: ComboboxComponent;
 126     @ViewChild('mergeProfileSelector')
 127         private mergeProfileSelector: ComboboxComponent;
 128     @ViewChild('fallThruMergeProfileSelector')
 129         private fallThruMergeProfileSelector: ComboboxComponent;
 130
 131     @ViewChild('dupeQueueAlert')
 132         private dupeQueueAlert: AlertDialogComponent;
 133
 134     constructor(
 135         private http: HttpClient,
 136         private toast: ToastService,
 137         private evt: EventService,
 138         private net: NetService,
 139         private auth: AuthService,
 140         private org: OrgService,
 141         private store: ServerStoreService,
 142         private vandelay: VandelayService
 143     ) {
 144         this.applyDefaults();
 145     }
 146
 147     applyDefaults() {
 148         this.minQualityRatio = 0;
 149         this.selectedBibSource = 1; // default to system local
 150         this.recordType = 'bib';
 151         this.bibTrashGroups = [];
 152         this.formTemplates = {};
 153
 154         if (this.vandelay.importSelection) {
 155
 156             if (!this.vandelay.importSelection.queue) {
 157                 // Incomplete import selection, clear it.
 158                 this.vandelay.importSelection = null;
 159                 return;
 160             }
 161
 162             const queue = this.vandelay.importSelection.queue;
 163             this.recordType = queue.queue_type();
 164             this.selectedMatchSet = queue.match_set();
 165
 166             // This will be propagated to selectedQueue as a combobox
 167             // entry via the combobox
 168             this.startQueueId = queue.id();
 169
 170             if (this.recordType === 'bib') {
 171                 this.selectedBucket = queue.match_bucket();
 172                 this.selectedHoldingsProfile = queue.item_attr_def();
 173             }
 174         }
 175     }
 176
 177     ngOnInit() {}
 178
 179     ngAfterViewInit() {
 180         this.loadStartupData();
 181     }
 182
 183     ngOnDestroy() {
 184         // Always clear the import selection when navigating away from
 185         // the import page.
 186         this.clearSelection();
 187     }
 188
 189     importSelection(): VandelayImportSelection {
 190         return this.vandelay.importSelection;
 191     }
 192
 193     loadStartupData(): Promise<any> {
 194         // Note displaying and manipulating a progress dialog inside
 195         // the AfterViewInit cycle leads to errors because the child
 196         // component is modifed after dirty checking.
 197
 198         const promises = [
 199             this.vandelay.getMergeProfiles(),
 200             this.vandelay.getAllQueues('bib'),
 201             this.vandelay.getAllQueues('authority'),
 202             this.vandelay.getMatchSets('bib'),
 203             this.vandelay.getMatchSets('authority'),
 204             this.vandelay.getBibBuckets(),
 205             this.vandelay.getBibSources(),
 206             this.vandelay.getItemImportDefs(),
 207             this.vandelay.getBibTrashGroups().then(
 208                 groups => this.bibTrashGroups = groups),
 209             this.org.settings(['vandelay.default_match_set']).then(
 210                 s => this.defaultMatchSet = s['vandelay.default_match_set']),
 211             this.loadTemplates()
 212         ];
 213
 214         return Promise.all(promises);
 215     }
 216
 217     loadTemplates() {
 218         this.store.getItem(TEMPLATE_SETTING_NAME).then(
 219             templates => {
 220                 this.formTemplates = templates || {};
 221
 222                 Object.keys(this.formTemplates).forEach(name => {
 223                     if (this.formTemplates[name].default) {
 224                         this.selectedTemplate = name;
 225                     }
 226                 });
 227             }
 228         );
 229     }
 230
 231     formatTemplateEntries(): ComboboxEntry[] {
 232         const entries = [];
 233
 234         Object.keys(this.formTemplates || {}).forEach(
 235             name => entries.push({id: name, label: name}));
 236
 237         return entries;
 238     }
 239
 240     // Format typeahead data sets
 241     formatEntries(etype: string): ComboboxEntry[] {
 242         const rtype = this.recordType;
 243         let list;
 244
 245         switch (etype) {
 246             case 'bibSources':
 247                 return (this.vandelay.bibSources || []).map(
 248                     s => {
 249                         return {id: s.id(), label: s.source()};
 250                     });
 251
 252             case 'bibBuckets':
 253                 list = this.vandelay.bibBuckets;
 254                 break;
 255
 256             case 'activeQueues':
 257                 list = (this.vandelay.allQueues[rtype] || [])
 258                         .filter(q => q.complete() === 'f');
 259                 break;
 260
 261             case 'matchSets':
 262                 list = this.vandelay.matchSets[rtype];
 263                 break;
 264
 265             case 'importItemDefs':
 266                 list = this.vandelay.importItemAttrDefs;
 267                 break;
 268
 269             case 'mergeProfiles':
 270                 list = this.vandelay.mergeProfiles;
 271                 break;
 272         }
 273
 274         return (list || []).map(item => {
 275             return {id: item.id(), label: item.name()};
 276         });
 277     }
 278
 279     selectEntry($event: ComboboxEntry, etype: string) {
 280         const id = $event ? $event.id : null;
 281
 282         switch (etype) {
 283             case 'recordType':
 284                 this.recordType = id;
 285                 break;
 286
 287             case 'bibSources':
 288                 this.selectedBibSource = id;
 289                 break;
 290
 291             case 'bibBuckets':
 292                 this.selectedBucket = id;
 293                 break;
 294
 295             case 'matchSets':
 296                 this.selectedMatchSet = id;
 297                 break;
 298
 299             case 'importItemDefs':
 300                 this.selectedHoldingsProfile = id;
 301                 break;
 302
 303             case 'mergeProfiles':
 304                 this.selectedMergeProfile = id;
 305                 break;
 306
 307             case 'FallThruMergeProfile':
 308                 this.selectedFallThruMergeProfile = id;
 309                 break;
 310         }
 311     }
 312
 313     fileSelected($event) {
 314        this.selectedFile = $event.target.files[0];
 315     }
 316
 317     // Required form data varies depending on context.
 318     hasNeededData(): boolean {
 319         if (this.vandelay.importSelection) {
 320             return this.importActionSelected();
 321         } else {
 322             return this.selectedQueue &&
 323                 Boolean(this.recordType) && Boolean(this.selectedFile);
 324         }
 325     }
 326
 327     importActionSelected(): boolean {
 328         return this.importNonMatching
 329             || this.mergeOnExact
 330             || this.mergeOnSingleMatch
 331             || this.mergeOnBestMatch;
 332     }
 333
 334     // 1. create queue if necessary
 335     // 2. upload MARC file
 336     // 3. Enqueue MARC records
 337     // 4. Import records
 338     upload() {
 339         this.sessionKey = null;
 340         this.showProgress = true;
 341         this.isUploading = true;
 342         this.uploadComplete = false;
 343         this.resetProgressBars();
 344
 345         this.resolveQueue()
 346         .then(
 347             queueId => {
 348                 this.activeQueueId = queueId;
 349                 return this.uploadFile();
 350             },
 351             err => Promise.reject('queue create failed')
 352         ).then(
 353             ok => this.processSpool(),
 354             err => Promise.reject('process spool failed')
 355         ).then(
 356             ok => this.importRecords(),
 357             err => Promise.reject('import records failed')
 358         ).then(
 359             ok => {
 360                 this.isUploading = false;
 361                 this.uploadComplete = true;
 362             },
 363             err => {
 364                 console.log('file upload failed: ', err);
 365                 this.isUploading = false;
 366                 this.resetProgressBars();
 367
 368             }
 369         );
 370     }
 371
 372     resetProgressBars() {
 373         this.uploadProgress.update({value: 0, max: 1});
 374         this.enqueueProgress.update({value: 0, max: 1});
 375         this.importProgress.update({value: 0, max: 1});
 376     }
 377
 378     // Extract selected queue ID or create a new queue when requested.
 379     resolveQueue(): Promise<number> {
 380
 381         if (this.selectedQueue.freetext) {
 382             // Free text queue selector means create a new entry.
 383             // TODO: first check for name dupes
 384
 385             return this.vandelay.createQueue(
 386                 this.selectedQueue.label,
 387                 this.recordType,
 388                 this.selectedHoldingsProfile,
 389                 this.selectedMatchSet,
 390                 this.selectedBucket
 391             ).then(
 392                 id => id,
 393                 err => {
 394                     const evt = this.evt.parse(err);
 395                     if (evt) {
 396                         if (evt.textcode.match(/QUEUE_EXISTS/)) {
 397                             this.dupeQueueAlert.open();
 398                         } else {
 399                             alert(evt); // server error
 400                         }
 401                     }
 402
 403                     return Promise.reject('Queue Create Failed');
 404                 }
 405             );
 406         } else {
 407             return Promise.resolve(this.selectedQueue.id);
 408         }
 409     }
 410
 411     uploadFile(): Promise<any> {
 412
 413         if (this.vandelay.importSelection) {
 414             // Nothing to upload when processing pre-queued records.
 415             return Promise.resolve();
 416         }
 417
 418         const formData: FormData = new FormData();
 419
 420         formData.append('ses', this.auth.token());
 421         formData.append('marc_upload',
 422             this.selectedFile, this.selectedFile.name);
 423
 424         if (this.selectedBibSource) {
 425             formData.append('bib_source', '' + this.selectedBibSource);
 426         }
 427
 428         const req = new HttpRequest('POST', VANDELAY_UPLOAD_PATH, formData,
 429             {reportProgress: true, responseType: 'text'});
 430
 431         return this.http.request(req).pipe(tap(
 432             evt => {
 433                 if (evt.type === HttpEventType.UploadProgress) {
 434                     this.uploadProgress.update(
 435                         {value: evt.loaded, max: evt.total});
 436
 437                 } else if (evt instanceof HttpResponse) {
 438                     this.sessionKey = evt.body as string;
 439                     console.log(
 440                         'Vandelay file uploaded OK with key ' + this.sessionKey);
 441                 }
 442             },
 443
 444             (err: HttpErrorResponse) => {
 445                 console.error(err);
 446                 this.toast.danger(err.error);
 447             }
 448         )).toPromise();
 449     }
 450
 451     processSpool():  Promise<any> {
 452
 453         if (this.vandelay.importSelection) {
 454             // Nothing to enqueue when processing pre-queued records
 455             return Promise.resolve();
 456         }
 457
 458         let spoolType = this.recordType;
 459         if (this.recordType === 'authority') {
 460             spoolType = 'auth';
 461         }
 462
 463         const method = `open-ils.vandelay.${spoolType}.process_spool`;
 464
 465         return new Promise((resolve, reject) => {
 466             this.net.request(
 467                 'open-ils.vandelay', method,
 468                 this.auth.token(), this.sessionKey, this.activeQueueId,
 469                 null, null, this.selectedBibSource,
 470                 (this.sessionName || null), true
 471             ).subscribe(
 472                 tracker => {
 473                     const e = this.evt.parse(tracker);
 474                     if (e) { console.error(e); return reject(); }
 475
 476                     // Spooling is in progress, track the results.
 477                     this.vandelay.pollSessionTracker(tracker.id())
 478                     .subscribe(
 479                         trkr => {
 480                             this.enqueueProgress.update({
 481                                 // enqueue API only tracks actions performed
 482                                 max: null,
 483                                 value: trkr.actions_performed()
 484                             });
 485                         },
 486                         err => { console.log(err); reject(); },
 487                         () => {
 488                             this.enqueueProgress.update({max: 1, value: 1});
 489                             resolve();
 490                         }
 491                     );
 492                 }
 493             );
 494         });
 495     }
 496
 497     importRecords(): Promise<any> {
 498
 499         if (!this.importActionSelected()) {
 500             return Promise.resolve();
 501         }
 502
 503         const selection = this.vandelay.importSelection;
 504
 505         if (selection && !selection.importQueue) {
 506             return this.importRecordQueue(selection.recordIds);
 507         } else {
 508             return this.importRecordQueue();
 509         }
 510     }
 511
 512     importRecordQueue(recIds?: number[]): Promise<any> {
 513         const rtype = this.recordType === 'bib' ? 'bib' : 'auth';
 514
 515         let method = `open-ils.vandelay.${rtype}_queue.import`;
 516         const options: ImportOptions = this.compileImportOptions();
 517
 518         let target: number | number[] = this.activeQueueId;
 519         if (recIds && recIds.length) {
 520             method = `open-ils.vandelay.${rtype}_record.list.import`;
 521             target = recIds;
 522         }
 523
 524         return new Promise((resolve, reject) => {
 525             this.net.request('open-ils.vandelay',
 526                 method, this.auth.token(), target, options)
 527             .subscribe(
 528                 tracker => {
 529                     const e = this.evt.parse(tracker);
 530                     if (e) { console.error(e); return reject(); }
 531
 532                     // Spooling is in progress, track the results.
 533                     this.vandelay.pollSessionTracker(tracker.id())
 534                     .subscribe(
 535                         trkr => {
 536                             this.importProgress.update({
 537                                 max: trkr.total_actions(),
 538                                 value: trkr.actions_performed()
 539                             });
 540                         },
 541                         err => { console.log(err); reject(); },
 542                         () => {
 543                             this.importProgress.update({max: 1, value: 1});
 544                             resolve();
 545                         }
 546                     );
 547                 }
 548             );
 549         });
 550     }
 551
 552     compileImportOptions(): ImportOptions {
 553
 554         const options: ImportOptions = {
 555             session_key: this.sessionKey,
 556             import_no_match: this.importNonMatching,
 557             auto_overlay_exact: this.mergeOnExact,
 558             auto_overlay_best_match: this.mergeOnBestMatch,
 559             auto_overlay_1match: this.mergeOnSingleMatch,
 560             opp_acq_copy_overlay: this.autoOverlayAcqCopies,
 561             merge_profile: this.selectedMergeProfile,
 562             fall_through_merge_profile: this.selectedFallThruMergeProfile,
 563             strip_field_groups: this.selectedTrashGroups,
 564             match_quality_ratio: this.minQualityRatio,
 565             exit_early: true
 566         };
 567
 568         if (this.vandelay.importSelection) {
 569             options.overlay_map = this.vandelay.importSelection.overlayMap;
 570         }
 571
 572         return options;
 573     }
 574
 575     clearSelection() {
 576         this.vandelay.importSelection = null;
 577         this.startQueueId = null;
 578     }
 579
 580     openQueue() {
 581         console.log('opening queue ' + this.activeQueueId);
 582     }
 583
 584     saveTemplate() {
 585
 586         const template = {};
 587         TEMPLATE_ATTRS.forEach(key => template[key] = this[key]);
 588
 589         console.debug('Saving import profile', template);
 590
 591         this.formTemplates[this.selectedTemplate] = template;
 592         return this.store.setItem(TEMPLATE_SETTING_NAME, this.formTemplates);
 593     }
 594
 595     markTemplateDefault() {
 596
 597         Object.keys(this.formTemplates).forEach(
 598             name => delete this.formTemplates.default
 599         );
 600
 601         this.formTemplates[this.selectedTemplate].default = true;
 602
 603         return this.store.setItem(TEMPLATE_SETTING_NAME, this.formTemplates);
 604     }
 605
 606     templateSelectorChange(entry: ComboboxEntry) {
 607
 608         if (!entry) {
 609             this.selectedTemplate = '';
 610             return;
 611         }
 612
 613         this.selectedTemplate = entry.label; // label == name
 614
 615         if (entry.freetext) {
 616             // User is entering a new template name.
 617             // Nothing to apply.
 618             return;
 619         }
 620
 621         // User selected an existing template, apply it to the form.
 622
 623         const template = this.formTemplates[entry.id];
 624
 625         // Copy the template values into "this"
 626         TEMPLATE_ATTRS.forEach(key => this[key] = template[key]);
 627
 628         // Some values must be manually passed to the combobox'es
 629
 630         this.recordTypeSelector.applyEntryId(this.recordType);
 631         this.bibSourceSelector.applyEntryId(this.selectedBibSource);
 632         this.matchSetSelector.applyEntryId(this.selectedMatchSet);
 633         this.holdingsProfileSelector
 634             .applyEntryId(this.selectedHoldingsProfile);
 635         this.mergeProfileSelector.applyEntryId(this.selectedMergeProfile);
 636         this.fallThruMergeProfileSelector
 637             .applyEntryId(this.selectedFallThruMergeProfile);
 638     }
 639
 640     deleteTemplate() {
 641         delete this.formTemplates[this.selectedTemplate];
 642         this.formTemplateSelector.selected = null;
 643         return this.store.setItem(TEMPLATE_SETTING_NAME, this.formTemplates);
 644     }
 645 }
 646