From aca9a4d5c7879737ca6fd26fa9b98bee413b93b5 Mon Sep 17 00:00:00 2001 From: Lebbeous Fogle-Weekley Date: Mon, 19 Sep 2011 13:19:22 -0400 Subject: [PATCH] Some EDI translation fixes for troublesome input characters This attempts to address enough of the issues in https://bugs.launchpad.net/evergreen/+bug/812593 to move forward. When the ? character appears in lineitem attributes for a PO that we send off for EDI translation, the translator fails if that ? is the last character of the string. This commit avoids that situation. This commit also removes any Unicode characters from the JEDI (the input to the translator process), as they're not going to make it to the EDIFACT output anyhow, and will just confound the translator. This commit also pre-escapes quotation marks in lineitem attributes so that we don't produce invalid JSON for our JEDI. This commit also destroys the death star. Seriously, the right way to fix some of the issues people have had is for somebody to learn Ruby and to address them either in edi_webrick.rb, in edi4r, or in openils-mapper (wherever the issues may be). Working on that. Signed-off-by: Lebbeous Fogle-Weekley Signed-off-by: Mike Rylander --- .../OpenILS/Application/Trigger/Reactor.pm | 52 ++++++++++++++----- Open-ILS/src/sql/Pg/950.data.seed-values.sql | 8 +-- .../Pg/upgrade/XXXX.data.jedi-template.sql | 14 +++++ Open-ILS/src/support-scripts/edi_pusher.pl | 8 ++- 4 files changed, 64 insertions(+), 18 deletions(-) create mode 100644 Open-ILS/src/sql/Pg/upgrade/XXXX.data.jedi-template.sql diff --git a/Open-ILS/src/perlmods/lib/OpenILS/Application/Trigger/Reactor.pm b/Open-ILS/src/perlmods/lib/OpenILS/Application/Trigger/Reactor.pm index 4214cabe6c..0f6262ff4d 100644 --- a/Open-ILS/src/perlmods/lib/OpenILS/Application/Trigger/Reactor.pm +++ b/Open-ILS/src/perlmods/lib/OpenILS/Application/Trigger/Reactor.pm @@ -17,6 +17,23 @@ sub NOOP_True { return 1 } sub NOOP_False { return 0 } +# To be used in two places within $_TT_helpers. Without putting the code out +# here, we can't really reuse it within that structure. +sub get_li_attr { + my $name = shift or return; # the first arg is always the name + my ($type, $attr) = (scalar(@_) == 1) ? (undef, $_[0]) : @_; + # if the next is the last, it's the attributes, otherwise type + # use Data::Dumper; $logger->warn("get_li_attr: " . Dumper($attr)); + ($name and @$attr) or return; + my $length; + $name =~ s/^(\D+)_(\d+)$/$1/ and $length = $2; + foreach (@$attr) { + $_->attr_name eq $name or next; + next if $length and $length != length($_->attr_value); + return $_->attr_value if (! $type) or $type eq $_->attr_type; + } + return; +} # helper functions inserted into the TT environment my $_TT_helpers = { @@ -134,20 +151,29 @@ my $_TT_helpers = { # helpers.get_li_attr('isbn_13', li.attributes) # returns matching line item attribute, or undef - get_li_attr => sub { - my $name = shift or return; # the first arg is always the name - my ($type, $attr) = (scalar(@_) == 1) ? (undef, $_[0]) : @_; - # if the next is the last, it's the attributes, otherwise type - # use Data::Dumper; $logger->warn("get_li_attr: " . Dumper($attr)); - ($name and @$attr) or return; - my $length; - $name =~ s/^(\D+)_(\d+)$/$1/ and $length = $2; - foreach (@$attr) { - $_->attr_name eq $name or next; - next if $length and $length != length($_->attr_value); - return $_->attr_value if (! $type) or $type eq $_->attr_type; + get_li_attr => \&get_li_attr, + + get_li_attr_jedi => sub { + my $value = get_li_attr(@_); + if ($value) { + # Here we can add any number of special case transformations to + # avoid problems with the EDI translator (or bad JSON). + + # The ? character, if in the final position of a string, breaks + # the translator. + or ' or : could be problematic, too. + if ($value =~ /[\?\+':]$/) { + chop $value; + } + + # Make sure any double quotation marks are escaped. + $value =~ s/"/\\"/g; + + # What the heck, get rid of [ ] too (although I couldn't get them + # to cause any problems for me. + $value =~ s/[\[\]]//g; } - return; + + return $value; }, get_queued_bib_attr => sub { diff --git a/Open-ILS/src/sql/Pg/950.data.seed-values.sql b/Open-ILS/src/sql/Pg/950.data.seed-values.sql index 8b8b29727a..ce0c907aba 100644 --- a/Open-ILS/src/sql/Pg/950.data.seed-values.sql +++ b/Open-ILS/src/sql/Pg/950.data.seed-values.sql @@ -7630,10 +7630,10 @@ END; ], "price":[% li.estimated_unit_price || '0.00' %], "desc":[ - {"BTI":"[% helpers.get_li_attr('title', '', li.attributes) %]"}, - {"BPU":"[% helpers.get_li_attr('publisher', '', li.attributes) %]"}, - {"BPD":"[% helpers.get_li_attr('pubdate', '', li.attributes) %]"}, - {"BPH":"[% helpers.get_li_attr('pagination','', li.attributes) %]"} + {"BTI":"[% helpers.get_li_attr_jedi('title', '', li.attributes) %]"}, + {"BPU":"[% helpers.get_li_attr_jedi('publisher', '', li.attributes) %]"}, + {"BPD":"[% helpers.get_li_attr_jedi('pubdate', '', li.attributes) %]"}, + {"BPH":"[% helpers.get_li_attr_jedi('pagination','', li.attributes) %]"} ], [%- ftx_vals = []; FOR note IN li.lineitem_notes; diff --git a/Open-ILS/src/sql/Pg/upgrade/XXXX.data.jedi-template.sql b/Open-ILS/src/sql/Pg/upgrade/XXXX.data.jedi-template.sql new file mode 100644 index 0000000000..e16f70004e --- /dev/null +++ b/Open-ILS/src/sql/Pg/upgrade/XXXX.data.jedi-template.sql @@ -0,0 +1,14 @@ +-- Evergreen DB patch XXXX.data.jedi-template.sql +-- +-- +BEGIN; + +-- check whether patch can be applied +SELECT evergreen.upgrade_deps_block_check('XXXX', :eg_version); + +UPDATE action_trigger.event_definition + SET template = + REPLACE(template, 'helpers.get_li_attr', 'helpers.get_li_attr_jedi') + WHERE id = 23; + +COMMIT; diff --git a/Open-ILS/src/support-scripts/edi_pusher.pl b/Open-ILS/src/support-scripts/edi_pusher.pl index 80e4e19ad1..ea9d63e1d8 100755 --- a/Open-ILS/src/support-scripts/edi_pusher.pl +++ b/Open-ILS/src/support-scripts/edi_pusher.pl @@ -155,7 +155,13 @@ foreach my $def (@$defs) { printf STDERR "ERROR: No edi_default account found for $logstr. File will not be sent!\n"; } - $message->jedi($event->template_output()->data); + my $jedi = $event->template_output()->data; + + # Crucial identifiers won't contain unicode characters, and EDIFACT + # (or at least our translator) generally can't handle them anyway. + $jedi =~ s/\\u[0-9a-f]{4}//g; + + $message->jedi($jedi); print "\ntarget->provider->edi_default->id: ", $target->provider->edi_default->id, "\n"; my $logstr2 = sprintf "event %s, PO %s, template_output %s", $_->{id}, $message->purchase_order, $event->template_output->id; -- 2.43.2