Open-ILS/src/c-apps/oils_sql.c

   1 /**
   2         @file oils_sql.c
   3         @brief Utility routines for translating JSON into SQL.
   4 */
   5
   6 #include <stdlib.h>
   7 #include <string.h>
   8 #include <time.h>
   9 #include <ctype.h>
  10 #include <dbi/dbi.h>
  11 #include "opensrf/utils.h"
  12 #include "opensrf/log.h"
  13 #include "opensrf/osrf_application.h"
  14 #include "openils/oils_utils.h"
  15 #include "openils/oils_sql.h"
  16
  17 // The next four macros are OR'd together as needed to form a set
  18 // of bitflags.  SUBCOMBO enables an extra pair of parentheses when
  19 // nesting one UNION, INTERSECT or EXCEPT inside another.
  20 // SUBSELECT tells us we're in a subquery, so don't add the
  21 // terminal semicolon yet.
  22 #define SUBCOMBO    8
  23 #define SUBSELECT   4
  24 #define DISABLE_I18N    2
  25 #define SELECT_DISTINCT 1
  26
  27 #define AND_OP_JOIN     0
  28 #define OR_OP_JOIN      1
  29
  30 struct ClassInfoStruct;
  31 typedef struct ClassInfoStruct ClassInfo;
  32
  33 #define ALIAS_STORE_SIZE 16
  34 #define CLASS_NAME_STORE_SIZE 16
  35
  36 struct ClassInfoStruct {
  37         char* alias;
  38         char* class_name;
  39         char* source_def;
  40         osrfHash* class_def;      // Points into IDL
  41         osrfHash* fields;         // Points into IDL
  42         osrfHash* links;          // Points into IDL
  43
  44         // The remaining members are private and internal.  Client code should not
  45         // access them directly.
  46
  47         ClassInfo* next;          // Supports linked list of joined classes
  48         int in_use;               // boolean
  49
  50         // We usually store the alias and class name in the following arrays, and
  51         // point the corresponding pointers at them.  When the string is too big
  52         // for the array (which will probably never happen in practice), we strdup it.
  53
  54         char alias_store[ ALIAS_STORE_SIZE + 1 ];
  55         char class_name_store[ CLASS_NAME_STORE_SIZE + 1 ];
  56 };
  57
  58 struct QueryFrameStruct;
  59 typedef struct QueryFrameStruct QueryFrame;
  60
  61 struct QueryFrameStruct {
  62         ClassInfo core;
  63         ClassInfo* join_list;  // linked list of classes joined to the core class
  64         QueryFrame* next;      // implements stack as linked list
  65         int in_use;            // boolean
  66 };
  67
  68 static int timeout_needs_resetting;
  69 static time_t time_next_reset;
  70
  71 static int verifyObjectClass ( osrfMethodContext*, const jsonObject* );
  72
  73 static void setXactId( osrfMethodContext* ctx );
  74 static inline const char* getXactId( osrfMethodContext* ctx );
  75 static inline void clearXactId( osrfMethodContext* ctx );
  76
  77 static jsonObject* doFieldmapperSearch ( osrfMethodContext* ctx, osrfHash* class_meta,
  78                 jsonObject* where_hash, jsonObject* query_hash, int* err );
  79 static jsonObject* oilsMakeFieldmapperFromResult( dbi_result, osrfHash* );
  80 static jsonObject* oilsMakeJSONFromResult( dbi_result );
  81
  82 static char* searchSimplePredicate ( const char* op, const char* class_alias,
  83                                 osrfHash* field, const jsonObject* node );
  84 static char* searchFunctionPredicate ( const char*, osrfHash*, const jsonObject*, const char* );
  85 static char* searchFieldTransform ( const char*, osrfHash*, const jsonObject* );
  86 static char* searchFieldTransformPredicate ( const ClassInfo*, osrfHash*, const jsonObject*,
  87                 const char* );
  88 static char* searchBETWEENPredicate ( const char*, osrfHash*, const jsonObject* );
  89 static char* searchINPredicate ( const char*, osrfHash*,
  90                                                                  jsonObject*, const char*, osrfMethodContext* );
  91 static char* searchPredicate ( const ClassInfo*, osrfHash*, jsonObject*, osrfMethodContext* );
  92 static char* searchJOIN ( const jsonObject*, const ClassInfo* left_info );
  93 static char* searchWHERE ( const jsonObject* search_hash, const ClassInfo*, int, osrfMethodContext* );
  94 static char* buildSELECT( const jsonObject*, jsonObject* rest_of_query,
  95         osrfHash* meta, osrfMethodContext* ctx );
  96 static char* buildOrderByFromArray( osrfMethodContext* ctx, const jsonObject* order_array );
  97
  98 char* buildQuery( osrfMethodContext* ctx, jsonObject* query, int flags );
  99
 100 char* SELECT ( osrfMethodContext*, jsonObject*, const jsonObject*, const jsonObject*,
 101         const jsonObject*, const jsonObject*, const jsonObject*, const jsonObject*, int );
 102
 103 static osrfStringArray* getPermLocationCache( osrfMethodContext*, const char* );
 104 static void setPermLocationCache( osrfMethodContext*, const char*, osrfStringArray* );
 105
 106 void userDataFree( void* );
 107 static void sessionDataFree( char*, void* );
 108 static void pcacheFree( char*, void* );
 109 static int obj_is_true( const jsonObject* obj );
 110 static const char* json_type( int code );
 111 static const char* get_primitive( osrfHash* field );
 112 static const char* get_datatype( osrfHash* field );
 113 static void pop_query_frame( void );
 114 static void push_query_frame( void );
 115 static int add_query_core( const char* alias, const char* class_name );
 116 static inline ClassInfo* search_alias( const char* target );
 117 static ClassInfo* search_all_alias( const char* target );
 118 static ClassInfo* add_joined_class( const char* alias, const char* classname );
 119 static void clear_query_stack( void );
 120
 121 static const jsonObject* verifyUserPCRUD( osrfMethodContext* );
 122 static const jsonObject* verifyUserPCRUDfull( osrfMethodContext*, int );
 123 static int verifyObjectPCRUD( osrfMethodContext*, osrfHash*, const jsonObject*, int );
 124 static const char* org_tree_root( osrfMethodContext* ctx );
 125 static jsonObject* single_hash( const char* key, const char* value );
 126
 127 static int child_initialized = 0;   /* boolean */
 128
 129 static dbi_conn writehandle; /* our MASTER db connection */
 130 static dbi_conn dbhandle; /* our CURRENT db connection */
 131 //static osrfHash * readHandles;
 132
 133 // The following points to the top of a stack of QueryFrames.  It's a little
 134 // confusing because the top level of the query is at the bottom of the stack.
 135 static QueryFrame* curr_query = NULL;
 136
 137 static dbi_conn writehandle; /* our MASTER db connection */
 138 static dbi_conn dbhandle; /* our CURRENT db connection */
 139 //static osrfHash * readHandles;
 140
 141 static int max_flesh_depth = 100;
 142
 143 static int perm_at_threshold = 5;
 144 static int enforce_pcrud = 0;     // Boolean
 145 static char* modulename = NULL;
 146
 147 int writeAuditInfo( osrfMethodContext* ctx, const char* user_id, const char* ws_id);
 148
 149 static char* _sanitize_tz_name( const char* tz );
 150 static char* _sanitize_savepoint_name( const char* sp );
 151
 152 /**
 153         @brief Connect to the database.
 154         @return A database connection if successful, or NULL if not.
 155 */
 156 dbi_conn oilsConnectDB( const char* mod_name ) {
 157
 158         osrfLogDebug( OSRF_LOG_MARK, "Attempting to initialize libdbi..." );
 159         if( dbi_initialize( NULL ) == -1 ) {
 160                 osrfLogError( OSRF_LOG_MARK, "Unable to initialize libdbi" );
 161                 return NULL;
 162         } else
 163                 osrfLogDebug( OSRF_LOG_MARK, "... libdbi initialized." );
 164
 165         char* driver = osrf_settings_host_value( "/apps/%s/app_settings/driver", mod_name );
 166         char* user   = osrf_settings_host_value( "/apps/%s/app_settings/database/user", mod_name );
 167         char* host   = osrf_settings_host_value( "/apps/%s/app_settings/database/host", mod_name );
 168         char* port   = osrf_settings_host_value( "/apps/%s/app_settings/database/port", mod_name );
 169         char* db     = osrf_settings_host_value( "/apps/%s/app_settings/database/db", mod_name );
 170         char* pw     = osrf_settings_host_value( "/apps/%s/app_settings/database/pw", mod_name );
 171         char* pg_app = osrf_settings_host_value( "/apps/%s/app_settings/database/application_name", mod_name );
 172
 173         osrfLogDebug( OSRF_LOG_MARK, "Attempting to load the database driver [%s]...", driver );
 174         dbi_conn handle = dbi_conn_new( driver );
 175
 176         if( !handle ) {
 177                 osrfLogError( OSRF_LOG_MARK, "Error loading database driver [%s]", driver );
 178                 return NULL;
 179         }
 180         osrfLogDebug( OSRF_LOG_MARK, "Database driver [%s] seems OK", driver );
 181
 182         osrfLogInfo(OSRF_LOG_MARK, "%s connecting to database.  host=%s, "
 183                 "port=%s, user=%s, db=%s", mod_name, host, port, user, db );
 184
 185         if( host )   dbi_conn_set_option( handle, "host", host );
 186         if( port )   dbi_conn_set_option_numeric( handle, "port", atoi( port ));
 187         if( user )   dbi_conn_set_option( handle, "username", user );
 188         if( pw )     dbi_conn_set_option( handle, "password", pw );
 189         if( db )     dbi_conn_set_option( handle, "dbname", db );
 190         if( pg_app ) dbi_conn_set_option( handle, "pgsql_application_name", pg_app );
 191
 192         free( user );
 193         free( host );
 194         free( port );
 195         free( db );
 196         free( pw );
 197         free( pg_app );
 198
 199         if( dbi_conn_connect( handle ) < 0 ) {
 200                 sleep( 1 );
 201                 if( dbi_conn_connect( handle ) < 0 ) {
 202                         const char* msg;
 203                         dbi_conn_error( handle, &msg );
 204                         osrfLogError( OSRF_LOG_MARK, "Error connecting to database: %s",
 205                                 msg ? msg : "(No description available)" );
 206                         return NULL;
 207                 }
 208         }
 209
 210         osrfLogInfo( OSRF_LOG_MARK, "%s successfully connected to the database", mod_name );
 211
 212         return handle;
 213 }
 214
 215 /**
 216         @brief Select some options.
 217         @param module_name: Name of the server.
 218         @param do_pcrud: Boolean.  True if we are to enforce PCRUD permissions.
 219
 220         This source file is used (at this writing) to implement three different servers:
 221         - open-ils.reporter-store
 222         - open-ils.pcrud
 223         - open-ils.cstore
 224
 225         These servers behave mostly the same, but they implement different combinations of
 226         methods, and open-ils.pcrud enforces a permissions scheme that the other two don't.
 227
 228         Here we use the server name in messages to identify which kind of server issued them.
 229         We use do_crud as a boolean to control whether or not to enforce the permissions scheme.
 230 */
 231 void oilsSetSQLOptions( const char* module_name, int do_pcrud, int flesh_depth ) {
 232         if( !module_name )
 233                 module_name = "open-ils.cstore";   // bulletproofing with a default
 234
 235         if( modulename )
 236                 free( modulename );
 237
 238         modulename = strdup( module_name );
 239         enforce_pcrud = do_pcrud;
 240         max_flesh_depth = flesh_depth;
 241 }
 242
 243 /**
 244         @brief Install a database connection.
 245         @param conn Pointer to a database connection.
 246
 247         In some contexts, @a conn may merely provide a driver so that we can process strings
 248         properly, without providing an open database connection.
 249 */
 250 void oilsSetDBConnection( dbi_conn conn ) {
 251         dbhandle = writehandle = conn;
 252 }
 253
 254 /**
 255         @brief Determine whether a database connection is alive.
 256         @param handle Handle for a database connection.
 257         @return 1 if the connection is alive, or zero if it isn't.
 258 */
 259 int oilsIsDBConnected( dbi_conn handle ) {
 260         // Do an innocuous SELECT.  If it succeeds, the database connection is still good.
 261         dbi_result result = dbi_conn_query( handle, "SELECT 1;" );
 262         if( result ) {
 263                 dbi_result_free( result );
 264                 return 1;
 265         } else {
 266                 // This is a terrible, horrible, no good, very bad kludge.
 267                 // Sometimes the SELECT 1 query fails, not because the database connection is dead,
 268                 // but because (due to a previous error) the database is ignoring all commands,
 269                 // even innocuous SELECTs, until the current transaction is rolled back.  The only
 270                 // known way to detect this condition via the dbi library is by looking at the error
 271                 // message.  This approach will break if the language or wording of the message ever
 272                 // changes.
 273                 // Note: the dbi_conn_ping function purports to determine whether the database
 274                 // connection is live, but at this writing this function is unreliable and useless.
 275                 static const char* ok_msg = "ERROR:  current transaction is aborted, commands "
 276                         "ignored until end of transaction block\n";
 277                 const char* msg;
 278                 dbi_conn_error( handle, &msg );
 279                 // Newer versions of dbi_conn_error return codes within the error msg.
 280                 // E.g. 3624914: ERROR:  current transaction is aborted, commands ignored until end of transaction block
 281                 // Substring test should work regardless.
 282                 const char* substr = strstr(msg, ok_msg);
 283                 if( substr == NULL ) {
 284                         osrfLogError( OSRF_LOG_MARK, "Database connection isn't working : %s", msg );
 285                         return 0;
 286                 } else
 287                         return 1;   // ignoring SELECT due to previous error; that's okay
 288         }
 289 }
 290
 291 /**
 292         @brief Get a table name, view name, or subquery for use in a FROM clause.
 293         @param class Pointer to the IDL class entry.
 294         @return A table name, a view name, or a subquery in parentheses.
 295
 296         In some cases the IDL defines a class, not with a table name or a view name, but with
 297         a SELECT statement, which may be used as a subquery.
 298 */
 299 char* oilsGetRelation( osrfHash* classdef ) {
 300
 301         char* source_def = NULL;
 302         const char* tabledef = osrfHashGet( classdef, "tablename" );
 303
 304         if( tabledef ) {
 305                 source_def = strdup( tabledef );   // Return the name of a table or view
 306         } else {
 307                 tabledef = osrfHashGet( classdef, "source_definition" );
 308                 if( tabledef ) {
 309                         // Return a subquery, enclosed in parentheses
 310                         source_def = safe_malloc( strlen( tabledef ) + 3 );
 311                         source_def[ 0 ] = '(';
 312                         strcpy( source_def + 1, tabledef );
 313                         strcat( source_def, ")" );
 314                 } else {
 315                         // Not found: return an error
 316                         const char* classname = osrfHashGet( classdef, "classname" );
 317                         if( !classname )
 318                                 classname = "???";
 319                         osrfLogError(
 320                                 OSRF_LOG_MARK,
 321                                 "%s ERROR No tablename or source_definition for class \"%s\"",
 322                                 modulename,
 323                                 classname
 324                         );
 325                 }
 326         }
 327
 328         return source_def;
 329 }
 330
 331 /**
 332         @brief Add datatypes from the database to the fields in the IDL.
 333         @param handle Handle for a database connection
 334         @return Zero if successful, or 1 upon error.
 335
 336         For each relevant class in the IDL: ask the database for the datatype of every field.
 337         In particular, determine which fields are text fields and which fields are numeric
 338         fields, so that we know whether to enclose their values in quotes.
 339 */
 340 int oilsExtendIDL( dbi_conn handle ) {
 341         osrfHashIterator* class_itr = osrfNewHashIterator( oilsIDL() );
 342         osrfHash* class = NULL;
 343         growing_buffer* query_buf = buffer_init( 64 );
 344         int results_found = 0;   // boolean
 345
 346         // For each class in the IDL...
 347         while( (class = osrfHashIteratorNext( class_itr ) ) ) {
 348                 const char* classname = osrfHashIteratorKey( class_itr );
 349                 osrfHash* fields = osrfHashGet( class, "fields" );
 350
 351                 // If the class is virtual, ignore it
 352                 if( str_is_true( osrfHashGet(class, "virtual") ) ) {
 353                         osrfLogDebug(OSRF_LOG_MARK, "Class %s is virtual, skipping", classname );
 354                         continue;
 355                 }
 356
 357                 char* tabledef = oilsGetRelation( class );
 358                 if( !tabledef )
 359                         continue;   // No such relation -- a query of it would be doomed to failure
 360
 361                 buffer_reset( query_buf );
 362                 buffer_fadd( query_buf, "SELECT * FROM %s AS x WHERE 1=0;", tabledef );
 363
 364                 free(tabledef );
 365
 366                 osrfLogDebug( OSRF_LOG_MARK, "%s Investigatory SQL = %s",
 367                                 modulename, OSRF_BUFFER_C_STR( query_buf ) );
 368
 369                 dbi_result result = dbi_conn_query( handle, OSRF_BUFFER_C_STR( query_buf ) );
 370                 if( result ) {
 371
 372                         results_found = 1;
 373                         int columnIndex = 1;
 374                         const char* columnName;
 375                         while( (columnName = dbi_result_get_field_name(result, columnIndex)) ) {
 376
 377                                 osrfLogInternal( OSRF_LOG_MARK, "Looking for column named [%s]...",
 378                                                 columnName );
 379
 380                                 /* fetch the fieldmapper index */
 381                                 osrfHash* _f = osrfHashGet(fields, columnName);
 382                                 if( _f ) {
 383
 384                                         osrfLogDebug(OSRF_LOG_MARK, "Found [%s] in IDL hash...", columnName);
 385
 386                                         /* determine the field type and storage attributes */
 387
 388                                         switch( dbi_result_get_field_type_idx( result, columnIndex )) {
 389
 390                                                 case DBI_TYPE_INTEGER : {
 391
 392                                                         if( !osrfHashGet(_f, "primitive") )
 393                                                                 osrfHashSet(_f, "number", "primitive");
 394
 395                                                         int attr = dbi_result_get_field_attribs_idx( result, columnIndex );
 396                                                         if( attr & DBI_INTEGER_SIZE8 )
 397                                                                 osrfHashSet( _f, "INT8", "datatype" );
 398                                                         else
 399                                                                 osrfHashSet( _f, "INT", "datatype" );
 400                                                         break;
 401                                                 }
 402                                                 case DBI_TYPE_DECIMAL :
 403                                                         if( !osrfHashGet( _f, "primitive" ))
 404                                                                 osrfHashSet( _f, "number", "primitive" );
 405
 406                                                         osrfHashSet( _f, "NUMERIC", "datatype" );
 407                                                         break;
 408
 409                                                 case DBI_TYPE_STRING :
 410                                                         if( !osrfHashGet( _f, "primitive" ))
 411                                                                 osrfHashSet( _f, "string", "primitive" );
 412
 413                                                         osrfHashSet( _f,"TEXT", "datatype" );
 414                                                         break;
 415
 416                                                 case DBI_TYPE_DATETIME :
 417                                                         if( !osrfHashGet( _f, "primitive" ))
 418                                                                 osrfHashSet( _f, "string", "primitive" );
 419
 420                                                         osrfHashSet( _f, "TIMESTAMP", "datatype" );
 421                                                         break;
 422
 423                                                 case DBI_TYPE_BINARY :
 424                                                         if( !osrfHashGet( _f, "primitive" ))
 425                                                                 osrfHashSet( _f, "string", "primitive" );
 426
 427                                                         osrfHashSet( _f, "BYTEA", "datatype" );
 428                                         }
 429
 430                                         osrfLogDebug(
 431                                                 OSRF_LOG_MARK,
 432                                                 "Setting [%s] to primitive [%s] and datatype [%s]...",
 433                                                 columnName,
 434                                                 osrfHashGet( _f, "primitive" ),
 435                                                 osrfHashGet( _f, "datatype" )
 436                                         );
 437                                 }
 438                                 ++columnIndex;
 439                         } // end while loop for traversing columns of result
 440                         dbi_result_free( result  );
 441                 } else {
 442                         const char* msg;
 443                         int errnum = dbi_conn_error( handle, &msg );
 444                         osrfLogDebug( OSRF_LOG_MARK, "No data found for class [%s]: %d, %s", classname,
 445                                 errnum, msg ? msg : "(No description available)" );
 446                         // We don't check the database connection here.  It's routine to get failures at
 447                         // this point; we routinely try to query tables that don't exist, because they
 448                         // are defined in the IDL but not in the database.
 449                 }
 450         } // end for each class in IDL
 451
 452         buffer_free( query_buf );
 453         osrfHashIteratorFree( class_itr );
 454         child_initialized = 1;
 455
 456         if( !results_found ) {
 457                 osrfLogError( OSRF_LOG_MARK,
 458                         "No results found for any class -- bad database connection?" );
 459                 return 1;
 460         } else if( ! oilsIsDBConnected( handle )) {
 461                 osrfLogError( OSRF_LOG_MARK,
 462                         "Unable to extend IDL: database connection isn't working" );
 463                 return 1;
 464         }
 465         else
 466                 return 0;
 467 }
 468
 469 /**
 470         @brief Free an osrfHash that stores a transaction ID.
 471         @param blob A pointer to the osrfHash to be freed, cast to a void pointer.
 472
 473         This function is a callback, to be called by the application session when it ends.
 474         The application session stores the osrfHash via an opaque pointer.
 475
 476         If the osrfHash contains an entry for the key "xact_id", it means that an
 477         uncommitted transaction is pending.  Roll it back.
 478 */
 479 void userDataFree( void* blob ) {
 480         osrfHash* hash = (osrfHash*) blob;
 481         if( osrfHashGet( hash, "xact_id" ) && writehandle ) {
 482                 if( !dbi_conn_query( writehandle, "ROLLBACK;" )) {
 483                         const char* msg;
 484                         int errnum = dbi_conn_error( writehandle, &msg );
 485                         osrfLogWarning( OSRF_LOG_MARK, "Unable to perform rollback: %d %s",
 486                                 errnum, msg ? msg : "(No description available)" );
 487                 };
 488         }
 489         if( writehandle ) {
 490                 if( !dbi_conn_query( writehandle, "SELECT auditor.clear_audit_info();" ) ) {
 491                         const char* msg;
 492                         int errnum = dbi_conn_error( writehandle, &msg );
 493                         osrfLogWarning( OSRF_LOG_MARK, "Unable to perform audit info clearing: %d %s",
 494                                 errnum, msg ? msg : "(No description available)" );
 495                 }
 496         }
 497
 498         osrfHashFree( hash );
 499 }
 500
 501 /**
 502         @name Managing session data
 503         @brief Maintain data stored via the userData pointer of the application session.
 504
 505         Currently, session-level data is stored in an osrfHash.  Other arrangements are
 506         possible, and some would be more efficient.  The application session calls a
 507         callback function to free userData before terminating.
 508
 509         Currently, the only data we store at the session level is the transaction id.  By this
 510         means we can ensure that any pending transactions are rolled back before the application
 511         session terminates.
 512 */
 513 /*@{*/
 514
 515 /**
 516         @brief Free an item in the application session's userData.
 517         @param key The name of a key for an osrfHash.
 518         @param item An opaque pointer to the item associated with the key.
 519
 520         We store an osrfHash as userData with the application session, and arrange (by
 521         installing userDataFree() as a different callback) for the session to free that
 522         osrfHash before terminating.
 523
 524         This function is a callback for freeing items in the osrfHash.  Currently we store
 525         two things:
 526         - Transaction id of a pending transaction; a character string.  Key: "xact_id".
 527         - Authkey; a character string.  Key: "authkey".
 528         - User object from the authentication server; a jsonObject.  Key: "user_login".
 529
 530         If we ever store anything else in userData, we will need to revisit this function so
 531         that it will free whatever else needs freeing.
 532 */
 533 static void sessionDataFree( char* key, void* item ) {
 534         if( !strcmp( key, "xact_id" ) || !strcmp( key, "authkey" ) || !strncmp( key, "rs_size_", 8) )
 535                 free( item );
 536         else if( !strcmp( key, "user_login" ) )
 537                 jsonObjectFree( (jsonObject*) item );
 538         else if( !strcmp( key, "pcache" ) )
 539                 osrfHashFree( (osrfHash*) item );
 540 }
 541
 542 static void pcacheFree( char* key, void* item ) {
 543         osrfStringArrayFree( (osrfStringArray*) item );
 544 }
 545
 546 /**
 547         @brief Initialize session cache.
 548         @param ctx Pointer to the method context.
 549
 550         Create a cache for the session by making the session's userData member point
 551         to an osrfHash instance.
 552 */
 553 static osrfHash* initSessionCache( osrfMethodContext* ctx ) {
 554         ctx->session->userData = osrfNewHash();
 555         osrfHashSetCallback( (osrfHash*) ctx->session->userData, &sessionDataFree );
 556         ctx->session->userDataFree = &userDataFree;
 557         return ctx->session->userData;
 558 }
 559
 560 /**
 561         @brief Save a transaction id.
 562         @param ctx Pointer to the method context.
 563
 564         Save the session_id of the current application session as a transaction id.
 565 */
 566 static void setXactId( osrfMethodContext* ctx ) {
 567         if( ctx && ctx->session ) {
 568                 osrfAppSession* session = ctx->session;
 569
 570                 osrfHash* cache = session->userData;
 571
 572                 // If the session doesn't already have a hash, create one.  Make sure
 573                 // that the application session frees the hash when it terminates.
 574                 if( NULL == cache )
 575                         cache = initSessionCache( ctx );
 576
 577                 // Save the transaction id in the hash, with the key "xact_id"
 578                 osrfHashSet( cache, strdup( session->session_id ), "xact_id" );
 579         }
 580 }
 581
 582 /**
 583         @brief Get the transaction ID for the current transaction, if any.
 584         @param ctx Pointer to the method context.
 585         @return Pointer to the transaction ID.
 586
 587         The return value points to an internal buffer, and will become invalid upon issuing
 588         a commit or rollback.
 589 */
 590 static inline const char* getXactId( osrfMethodContext* ctx ) {
 591         if( ctx && ctx->session && ctx->session->userData )
 592                 return osrfHashGet( (osrfHash*) ctx->session->userData, "xact_id" );
 593         else
 594                 return NULL;
 595 }
 596
 597 /**
 598         @brief Clear the current transaction id.
 599         @param ctx Pointer to the method context.
 600 */
 601 static inline void clearXactId( osrfMethodContext* ctx ) {
 602         if( ctx && ctx->session && ctx->session->userData )
 603                 osrfHashRemove( ctx->session->userData, "xact_id" );
 604 }
 605 /*@}*/
 606
 607 /**
 608         @brief Stash the location for a particular perm in the sessionData cache
 609         @param ctx Pointer to the method context.
 610         @param perm Name of the permission we're looking at
 611         @param array StringArray of perm location ids
 612 */
 613 static void setPermLocationCache( osrfMethodContext* ctx, const char* perm, osrfStringArray* locations ) {
 614         if( ctx && ctx->session ) {
 615                 osrfAppSession* session = ctx->session;
 616
 617                 osrfHash* cache = session->userData;
 618
 619                 // If the session doesn't already have a hash, create one.  Make sure
 620                 // that the application session frees the hash when it terminates.
 621                 if( NULL == cache )
 622                         cache = initSessionCache( ctx );
 623
 624                 osrfHash* pcache = osrfHashGet(cache, "pcache");
 625
 626                 if( NULL == pcache ) {
 627                         pcache = osrfNewHash();
 628                         osrfHashSetCallback( pcache, &pcacheFree );
 629                         osrfHashSet( cache, pcache, "pcache" );
 630                 }
 631
 632                 if( perm && locations )
 633                         osrfHashSet( pcache, locations, strdup(perm) );
 634         }
 635 }
 636
 637 /**
 638         @brief Grab stashed location for a particular perm in the sessionData cache
 639         @param ctx Pointer to the method context.
 640         @param perm Name of the permission we're looking at
 641 */
 642 static osrfStringArray* getPermLocationCache( osrfMethodContext* ctx, const char* perm ) {
 643         if( ctx && ctx->session ) {
 644                 osrfAppSession* session = ctx->session;
 645                 osrfHash* cache = session->userData;
 646                 if( cache ) {
 647                         osrfHash* pcache = osrfHashGet(cache, "pcache");
 648                         if( pcache ) {
 649                                 return osrfHashGet( pcache, perm );
 650                         }
 651                 }
 652         }
 653
 654         return NULL;
 655 }
 656
 657 /**
 658         @brief Save the user's login in the userData for the current application session.
 659         @param ctx Pointer to the method context.
 660         @param user_login Pointer to the user login object to be cached (we cache the original,
 661         not a copy of it).
 662
 663         If @a user_login is NULL, remove the user login if one is already cached.
 664 */
 665 static void setUserLogin( osrfMethodContext* ctx, jsonObject* user_login ) {
 666         if( ctx && ctx->session ) {
 667                 osrfAppSession* session = ctx->session;
 668
 669                 osrfHash* cache = session->userData;
 670
 671                 // If the session doesn't already have a hash, create one.  Make sure
 672                 // that the application session frees the hash when it terminates.
 673                 if( NULL == cache )
 674                         cache = initSessionCache( ctx );
 675
 676                 if( user_login )
 677                         osrfHashSet( cache, user_login, "user_login" );
 678                 else
 679                         osrfHashRemove( cache, "user_login" );
 680         }
 681 }
 682
 683 /**
 684         @brief Get the user login object for the current application session, if any.
 685         @param ctx Pointer to the method context.
 686         @return Pointer to the user login object if found; otherwise NULL.
 687
 688         The user login object was returned from the authentication server, and then cached so
 689         we don't have to call the authentication server again for the same user.
 690 */
 691 static const jsonObject* getUserLogin( osrfMethodContext* ctx ) {
 692         if( ctx && ctx->session && ctx->session->userData )
 693                 return osrfHashGet( (osrfHash*) ctx->session->userData, "user_login" );
 694         else
 695                 return NULL;
 696 }
 697
 698 /**
 699         @brief Save a copy of an authkey in the userData of the current application session.
 700         @param ctx Pointer to the method context.
 701         @param authkey The authkey to be saved.
 702
 703         If @a authkey is NULL, remove the authkey if one is already cached.
 704 */
 705 static void setAuthkey( osrfMethodContext* ctx, const char* authkey ) {
 706         if( ctx && ctx->session && authkey ) {
 707                 osrfAppSession* session = ctx->session;
 708                 osrfHash* cache = session->userData;
 709
 710                 // If the session doesn't already have a hash, create one.  Make sure
 711                 // that the application session frees the hash when it terminates.
 712                 if( NULL == cache )
 713                         cache = initSessionCache( ctx );
 714
 715                 // Save the transaction id in the hash, with the key "xact_id"
 716                 if( authkey && *authkey )
 717                         osrfHashSet( cache, strdup( authkey ), "authkey" );
 718                 else
 719                         osrfHashRemove( cache, "authkey" );
 720         }
 721 }
 722
 723 /**
 724         @brief Reset the login timeout.
 725         @param authkey The authentication key for the current login session.
 726         @param now The current time.
 727         @return Zero if successful, or 1 if not.
 728
 729         Tell the authentication server to reset the timeout so that the login session won't
 730         expire for a while longer.
 731
 732         We could dispense with the @a now parameter by calling time().  But we just called
 733         time() in order to decide whether to reset the timeout, so we might as well reuse
 734         the result instead of calling time() again.
 735 */
 736 static int reset_timeout( const char* authkey, time_t now ) {
 737         jsonObject* auth_object = jsonNewObject( authkey );
 738
 739         // Ask the authentication server to reset the timeout.  It returns an event
 740         // indicating success or failure.
 741         jsonObject* result = oilsUtilsQuickReq( "open-ils.auth",
 742                 "open-ils.auth.session.reset_timeout", auth_object );
 743         jsonObjectFree( auth_object );
 744
 745         if( !result || result->type != JSON_HASH ) {
 746                 osrfLogError( OSRF_LOG_MARK,
 747                          "Unexpected object type receieved from open-ils.auth.session.reset_timeout" );
 748                 jsonObjectFree( result );
 749                 return 1;       // Not the right sort of object returned
 750         }
 751
 752         const jsonObject* ilsevent = jsonObjectGetKeyConst( result, "ilsevent" );
 753         if( !ilsevent || ilsevent->type != JSON_NUMBER ) {
 754                 osrfLogError( OSRF_LOG_MARK, "ilsevent is absent or malformed" );
 755                 jsonObjectFree( result );
 756                 return 1;    // Return code from method not available
 757         }
 758
 759         if( jsonObjectGetNumber( ilsevent ) != 0.0 ) {
 760                 const char* desc = jsonObjectGetString( jsonObjectGetKeyConst( result, "desc" ));
 761                 if( !desc )
 762                         desc = "(No reason available)";    // failsafe; shouldn't happen
 763                 osrfLogInfo( OSRF_LOG_MARK, "Failure to reset timeout: %s", desc );
 764                 jsonObjectFree( result );
 765                 return 1;
 766         }
 767
 768         // Revise our local proxy for the timeout deadline
 769         // by a smallish fraction of the timeout interval
 770         const char* timeout = jsonObjectGetString( jsonObjectGetKeyConst( result, "payload" ));
 771         if( !timeout )
 772                 timeout = "1";   // failsafe; shouldn't happen
 773         time_next_reset = now + atoi( timeout ) / 15;
 774
 775         jsonObjectFree( result );
 776         return 0;     // Successfully reset timeout
 777 }
 778
 779 /**
 780         @brief Get the authkey string for the current application session, if any.
 781         @param ctx Pointer to the method context.
 782         @return Pointer to the cached authkey if found; otherwise NULL.
 783
 784         If present, the authkey string was cached from a previous method call.
 785 */
 786 static const char* getAuthkey( osrfMethodContext* ctx ) {
 787         if( ctx && ctx->session && ctx->session->userData ) {
 788                 const char* authkey = osrfHashGet( (osrfHash*) ctx->session->userData, "authkey" );
 789         // LFW recent changes mean the userData hash gets set up earlier, but
 790         // doesn't necessarily have an authkey yet
 791         if (!authkey)
 792             return NULL;
 793
 794                 // Possibly reset the authentication timeout to keep the login alive.  We do so
 795                 // no more than once per method call, and not at all if it has been only a short
 796                 // time since the last reset.
 797
 798                 // Here we reset explicitly, if at all.  We also implicitly reset the timeout
 799                 // whenever we call the "open-ils.auth.session.retrieve" method.
 800                 if( timeout_needs_resetting ) {
 801                         time_t now = time( NULL );
 802                         if( now >= time_next_reset && reset_timeout( authkey, now ) )
 803                                 authkey = NULL;    // timeout has apparently expired already
 804                 }
 805
 806                 timeout_needs_resetting = 0;
 807                 return authkey;
 808         }
 809         else
 810                 return NULL;
 811 }
 812
 813 /**
 814         @brief Implement the transaction.begin method.
 815         @param ctx Pointer to the method context.
 816         @return Zero if successful, or -1 upon error.
 817
 818         Start a transaction.  Save a transaction ID for future reference.
 819
 820         Method parameters:
 821         - authkey (PCRUD only)
 822
 823         Return to client: Transaction ID
 824 */
 825 int beginTransaction( osrfMethodContext* ctx ) {
 826         if(osrfMethodVerifyContext( ctx )) {
 827                 osrfLogError( OSRF_LOG_MARK,  "Invalid method context" );
 828                 return -1;
 829         }
 830
 831         const char* tz = _sanitize_tz_name(ctx->session->session_tz);
 832
 833         if( enforce_pcrud ) {
 834                 timeout_needs_resetting = 1;
 835                 const jsonObject* user = verifyUserPCRUD( ctx );
 836                 if( !user )
 837                         return -1;
 838         }
 839
 840         dbi_result result = dbi_conn_query( writehandle, "START TRANSACTION;" );
 841         if( !result ) {
 842                 const char* msg;
 843                 int errnum = dbi_conn_error( writehandle, &msg );
 844                 osrfLogError( OSRF_LOG_MARK, "%s: Error starting transaction: %d %s",
 845                         modulename, errnum, msg ? msg : "(No description available)" );
 846                 osrfAppSessionStatus( ctx->session, OSRF_STATUS_INTERNALSERVERERROR,
 847                         "osrfMethodException", ctx->request, "Error starting transaction" );
 848                 if( !oilsIsDBConnected( writehandle ))
 849                         osrfAppSessionPanic( ctx->session );
 850                 return -1;
 851         } else {
 852                 dbi_result_free( result );
 853                 setXactId( ctx );
 854                 jsonObject* ret = jsonNewObject( getXactId( ctx ) );
 855                 osrfAppRespondComplete( ctx, ret );
 856                 jsonObjectFree( ret );
 857
 858         }
 859
 860         if (tz) {
 861                 setenv("TZ",tz,1);
 862                 tzset();
 863                 dbi_result tz_res = dbi_conn_queryf( writehandle, "SET LOCAL timezone TO '%s'; -- cstore", tz );
 864                 if( !tz_res ) {
 865                         osrfLogError( OSRF_LOG_MARK, "%s: Error setting timezone %s", modulename, tz);
 866                         if( !oilsIsDBConnected( writehandle )) {
 867                                 osrfAppSessionPanic( ctx->session );
 868                                 return -1;
 869                         }
 870                 } else {
 871                         dbi_result_free( tz_res );
 872                 }
 873         } else {
 874                 unsetenv("TZ");
 875                 tzset();
 876                 dbi_result res = dbi_conn_queryf( writehandle, "SET timezone TO DEFAULT; -- no tz" );
 877                 if( !res ) {
 878                         osrfLogError( OSRF_LOG_MARK, "%s: Error resetting timezone", modulename);
 879                         if( !oilsIsDBConnected( writehandle )) {
 880                                 osrfAppSessionPanic( ctx->session );
 881                                 return -1;
 882                         }
 883                 } else {
 884                         dbi_result_free( res );
 885                 }
 886         }
 887
 888         return 0;
 889 }
 890
 891 /**
 892         @brief Implement the savepoint.set method.
 893         @param ctx Pointer to the method context.
 894         @return Zero if successful, or -1 if not.
 895
 896         Issue a SAVEPOINT to the database server.
 897
 898         Method parameters:
 899         - authkey (PCRUD only)
 900         - savepoint name
 901
 902         Return to client: Savepoint name
 903 */
 904 int setSavepoint( osrfMethodContext* ctx ) {
 905         if(osrfMethodVerifyContext( ctx )) {
 906                 osrfLogError( OSRF_LOG_MARK,  "Invalid method context" );
 907                 return -1;
 908         }
 909
 910         int spNamePos = 0;
 911         if( enforce_pcrud ) {
 912                 spNamePos = 1;
 913                 timeout_needs_resetting = 1;
 914                 const jsonObject* user = verifyUserPCRUD( ctx );
 915                 if( !user )
 916                         return -1;
 917         }
 918
 919         // Verify that a transaction is pending
 920         const char* trans_id = getXactId( ctx );
 921         if( NULL == trans_id ) {
 922                 osrfAppSessionStatus(
 923                         ctx->session,
 924                         OSRF_STATUS_INTERNALSERVERERROR,
 925                         "osrfMethodException",
 926                         ctx->request,
 927                         "No active transaction -- required for savepoints"
 928                 );
 929                 return -1;
 930         }
 931
 932         // Get the savepoint name from the method params
 933         const char* spName = jsonObjectGetString( jsonObjectGetIndex(ctx->params, spNamePos) );
 934
 935         if (!spName) {
 936                 osrfLogWarning(OSRF_LOG_MARK, "savepoint.set called with no name");
 937                 return -1;
 938         }
 939
 940         char *safeSpName = _sanitize_savepoint_name( spName );
 941
 942         dbi_result result = dbi_conn_queryf( writehandle, "SAVEPOINT \"%s\";", safeSpName );
 943         free( safeSpName );
 944         if( !result ) {
 945                 const char* msg;
 946                 int errnum = dbi_conn_error( writehandle, &msg );
 947                 osrfLogError(
 948                         OSRF_LOG_MARK,
 949                         "%s: Error creating savepoint %s in transaction %s: %d %s",
 950                         modulename,
 951                         spName,
 952                         trans_id,
 953                         errnum,
 954                         msg ? msg : "(No description available)"
 955                 );
 956                 osrfAppSessionStatus( ctx->session, OSRF_STATUS_INTERNALSERVERERROR,
 957                         "osrfMethodException", ctx->request, "Error creating savepoint" );
 958                 if( !oilsIsDBConnected( writehandle ))
 959                         osrfAppSessionPanic( ctx->session );
 960                 return -1;
 961         } else {
 962                 dbi_result_free( result );
 963                 jsonObject* ret = jsonNewObject( spName );
 964                 osrfAppRespondComplete( ctx, ret );
 965                 jsonObjectFree( ret  );
 966                 return 0;
 967         }
 968 }
 969
 970 /**
 971         @brief Implement the savepoint.release method.
 972         @param ctx Pointer to the method context.
 973         @return Zero if successful, or -1 if not.
 974
 975         Issue a RELEASE SAVEPOINT to the database server.
 976
 977         Method parameters:
 978         - authkey (PCRUD only)
 979         - savepoint name
 980
 981         Return to client: Savepoint name
 982 */
 983 int releaseSavepoint( osrfMethodContext* ctx ) {
 984         if(osrfMethodVerifyContext( ctx )) {
 985                 osrfLogError( OSRF_LOG_MARK,  "Invalid method context" );
 986                 return -1;
 987         }
 988
 989         int spNamePos = 0;
 990         if( enforce_pcrud ) {
 991                 spNamePos = 1;
 992                 timeout_needs_resetting = 1;
 993                 const jsonObject* user = verifyUserPCRUD( ctx );
 994                 if(  !user )
 995                         return -1;
 996         }
 997
 998         // Verify that a transaction is pending
 999         const char* trans_id = getXactId( ctx );
1000         if( NULL == trans_id ) {
1001                 osrfAppSessionStatus(
1002                         ctx->session,
1003                         OSRF_STATUS_INTERNALSERVERERROR,
1004                         "osrfMethodException",
1005                         ctx->request,
1006                         "No active transaction -- required for savepoints"
1007                 );
1008                 return -1;
1009         }
1010
1011         // Get the savepoint name from the method params
1012         const char* spName = jsonObjectGetString( jsonObjectGetIndex(ctx->params, spNamePos) );
1013
1014         if (!spName) {
1015                 osrfLogWarning(OSRF_LOG_MARK, "savepoint.release called with no name");
1016                 return -1;
1017         }
1018
1019         char *safeSpName = _sanitize_savepoint_name( spName );
1020
1021         dbi_result result = dbi_conn_queryf( writehandle, "RELEASE SAVEPOINT \"%s\";", safeSpName );
1022         free( safeSpName );
1023         if( !result ) {
1024                 const char* msg;
1025                 int errnum = dbi_conn_error( writehandle, &msg );
1026                 osrfLogError(
1027                         OSRF_LOG_MARK,
1028                         "%s: Error releasing savepoint %s in transaction %s: %d %s",
1029                         modulename,
1030                         spName,
1031                         trans_id,
1032                         errnum,
1033                         msg ? msg : "(No description available)"
1034                 );
1035                 osrfAppSessionStatus( ctx->session, OSRF_STATUS_INTERNALSERVERERROR,
1036                         "osrfMethodException", ctx->request, "Error releasing savepoint" );
1037                 if( !oilsIsDBConnected( writehandle ))
1038                         osrfAppSessionPanic( ctx->session );
1039                 return -1;
1040         } else {
1041                 dbi_result_free( result );
1042                 jsonObject* ret = jsonNewObject( spName );
1043                 osrfAppRespondComplete( ctx, ret );
1044                 jsonObjectFree( ret );
1045                 return 0;
1046         }
1047 }
1048
1049 /**
1050         @brief Implement the savepoint.rollback method.
1051         @param ctx Pointer to the method context.
1052         @return Zero if successful, or -1 if not.
1053
1054         Issue a ROLLBACK TO SAVEPOINT to the database server.
1055
1056         Method parameters:
1057         - authkey (PCRUD only)
1058         - savepoint name
1059
1060         Return to client: Savepoint name
1061 */
1062 int rollbackSavepoint( osrfMethodContext* ctx ) {
1063         if(osrfMethodVerifyContext( ctx )) {
1064                 osrfLogError( OSRF_LOG_MARK,  "Invalid method context" );
1065                 return -1;
1066         }
1067
1068         int spNamePos = 0;
1069         if( enforce_pcrud ) {
1070                 spNamePos = 1;
1071                 timeout_needs_resetting = 1;
1072                 const jsonObject* user = verifyUserPCRUD( ctx );
1073                 if( !user )
1074                         return -1;
1075         }
1076
1077         // Verify that a transaction is pending
1078         const char* trans_id = getXactId( ctx );
1079         if( NULL == trans_id ) {
1080                 osrfAppSessionStatus(
1081                         ctx->session,
1082                         OSRF_STATUS_INTERNALSERVERERROR,
1083                         "osrfMethodException",
1084                         ctx->request,
1085                         "No active transaction -- required for savepoints"
1086                 );
1087                 return -1;
1088         }
1089
1090         // Get the savepoint name from the method params
1091         const char* spName = jsonObjectGetString( jsonObjectGetIndex(ctx->params, spNamePos) );
1092
1093         if (!spName) {
1094                 osrfLogWarning(OSRF_LOG_MARK, "savepoint.rollback called with no name");
1095                 return -1;
1096         }
1097
1098         char *safeSpName = _sanitize_savepoint_name( spName );
1099
1100         dbi_result result = dbi_conn_queryf( writehandle, "ROLLBACK TO SAVEPOINT \"%s\";", safeSpName );
1101         free( safeSpName );
1102         if( !result ) {
1103                 const char* msg;
1104                 int errnum = dbi_conn_error( writehandle, &msg );
1105                 osrfLogError(
1106                         OSRF_LOG_MARK,
1107                         "%s: Error rolling back savepoint %s in transaction %s: %d %s",
1108                         modulename,
1109                         spName,
1110                         trans_id,
1111                         errnum,
1112                         msg ? msg : "(No description available)"
1113                 );
1114                 osrfAppSessionStatus( ctx->session, OSRF_STATUS_INTERNALSERVERERROR,
1115                         "osrfMethodException", ctx->request, "Error rolling back savepoint" );
1116                 if( !oilsIsDBConnected( writehandle ))
1117                         osrfAppSessionPanic( ctx->session );
1118                 return -1;
1119         } else {
1120                 dbi_result_free( result );
1121                 jsonObject* ret = jsonNewObject( spName );
1122                 osrfAppRespondComplete( ctx, ret );
1123                 jsonObjectFree( ret );
1124                 return 0;
1125         }
1126 }
1127
1128 /**
1129         @brief Implement the transaction.commit method.
1130         @param ctx Pointer to the method context.
1131         @return Zero if successful, or -1 if not.
1132
1133         Issue a COMMIT to the database server.
1134
1135         Method parameters:
1136         - authkey (PCRUD only)
1137
1138         Return to client: Transaction ID.
1139 */
1140 int commitTransaction( osrfMethodContext* ctx ) {
1141         if(osrfMethodVerifyContext( ctx )) {
1142                 osrfLogError( OSRF_LOG_MARK, "Invalid method context" );
1143                 return -1;
1144         }
1145
1146         if( enforce_pcrud ) {
1147                 timeout_needs_resetting = 1;
1148                 const jsonObject* user = verifyUserPCRUD( ctx );
1149                 if( !user )
1150                         return -1;
1151         }
1152
1153         // Verify that a transaction is pending
1154         const char* trans_id = getXactId( ctx );
1155         if( NULL == trans_id ) {
1156                 osrfAppSessionStatus( ctx->session, OSRF_STATUS_INTERNALSERVERERROR,
1157                                 "osrfMethodException", ctx->request, "No active transaction to commit" );
1158                 return -1;
1159         }
1160
1161         dbi_result result = dbi_conn_query( writehandle, "COMMIT;" );
1162         if( !result ) {
1163                 const char* msg;
1164                 int errnum = dbi_conn_error( writehandle, &msg );
1165                 osrfLogError( OSRF_LOG_MARK, "%s: Error committing transaction: %d %s",
1166                         modulename, errnum, msg ? msg : "(No description available)" );
1167                 osrfAppSessionStatus( ctx->session, OSRF_STATUS_INTERNALSERVERERROR,
1168                         "osrfMethodException", ctx->request, "Error committing transaction" );
1169                 if( !oilsIsDBConnected( writehandle ))
1170                         osrfAppSessionPanic( ctx->session );
1171                 return -1;
1172         } else {
1173                 dbi_result_free( result );
1174                 jsonObject* ret = jsonNewObject( trans_id );
1175                 osrfAppRespondComplete( ctx, ret );
1176                 jsonObjectFree( ret );
1177                 clearXactId( ctx );
1178                 return 0;
1179         }
1180 }
1181
1182 /**
1183         @brief Implement the transaction.rollback method.
1184         @param ctx Pointer to the method context.
1185         @return Zero if successful, or -1 if not.
1186
1187         Issue a ROLLBACK to the database server.
1188
1189         Method parameters:
1190         - authkey (PCRUD only)
1191
1192         Return to client: Transaction ID
1193 */
1194 int rollbackTransaction( osrfMethodContext* ctx ) {
1195         if( osrfMethodVerifyContext( ctx )) {
1196                 osrfLogError( OSRF_LOG_MARK,  "Invalid method context" );
1197                 return -1;
1198         }
1199
1200         if( enforce_pcrud ) {
1201                 timeout_needs_resetting = 1;
1202                 const jsonObject* user = verifyUserPCRUD( ctx );
1203                 if( !user )
1204                         return -1;
1205         }
1206
1207         // Verify that a transaction is pending
1208         const char* trans_id = getXactId( ctx );
1209         if( NULL == trans_id ) {
1210                 osrfAppSessionStatus( ctx->session, OSRF_STATUS_INTERNALSERVERERROR,
1211                                 "osrfMethodException", ctx->request, "No active transaction to roll back" );
1212                 return -1;
1213         }
1214
1215         dbi_result result = dbi_conn_query( writehandle, "ROLLBACK;" );
1216         if( !result ) {
1217                 const char* msg;
1218                 int errnum = dbi_conn_error( writehandle, &msg );
1219                 osrfLogError( OSRF_LOG_MARK, "%s: Error rolling back transaction: %d %s",
1220                         modulename, errnum, msg ? msg : "(No description available)" );
1221                 osrfAppSessionStatus( ctx->session, OSRF_STATUS_INTERNALSERVERERROR,
1222                         "osrfMethodException", ctx->request, "Error rolling back transaction" );
1223                 if( !oilsIsDBConnected( writehandle ))
1224                         osrfAppSessionPanic( ctx->session );
1225                 return -1;
1226         } else {
1227                 dbi_result_free( result );
1228                 jsonObject* ret = jsonNewObject( trans_id );
1229                 osrfAppRespondComplete( ctx, ret );
1230                 jsonObjectFree( ret );
1231                 clearXactId( ctx );
1232                 return 0;
1233         }
1234 }
1235
1236 /**
1237         @brief Implement the "search" method.
1238         @param ctx Pointer to the method context.
1239         @return Zero if successful, or -1 if not.
1240
1241         Method parameters:
1242         - authkey (PCRUD only)
1243         - WHERE clause, as jsonObject
1244         - Other SQL clause(s), as a JSON_HASH: joins, SELECT list, LIMIT, etc.
1245
1246         Return to client: rows of the specified class that satisfy a specified WHERE clause.
1247         Optionally flesh linked fields.
1248 */
1249 int doSearch( osrfMethodContext* ctx ) {
1250         if( osrfMethodVerifyContext( ctx )) {
1251                 osrfLogError( OSRF_LOG_MARK, "Invalid method context" );
1252                 return -1;
1253         }
1254
1255         if( enforce_pcrud )
1256                 timeout_needs_resetting = 1;
1257
1258         jsonObject* where_clause;
1259         jsonObject* rest_of_query;
1260
1261         if( enforce_pcrud ) {
1262                 where_clause  = jsonObjectGetIndex( ctx->params, 1 );
1263                 rest_of_query = jsonObjectGetIndex( ctx->params, 2 );
1264         } else {
1265                 where_clause  = jsonObjectGetIndex( ctx->params, 0 );
1266                 rest_of_query = jsonObjectGetIndex( ctx->params, 1 );
1267         }
1268
1269         if( !where_clause ) {
1270                 osrfLogError( OSRF_LOG_MARK, "No WHERE clause parameter supplied" );
1271                 return -1;
1272         }
1273
1274         // Get the class metadata
1275         osrfHash* method_meta = (osrfHash*) ctx->method->userData;
1276         osrfHash* class_meta = osrfHashGet( method_meta, "class" );
1277
1278         // Do the query
1279         int err = 0;
1280         jsonObject* obj = doFieldmapperSearch( ctx, class_meta, where_clause, rest_of_query, &err );
1281         if( err ) {
1282                 osrfAppRespondComplete( ctx, NULL );
1283                 return -1;
1284         }
1285
1286         // doFieldmapperSearch() now takes care of our responding for us
1287 //      // Return each row to the client
1288 //      jsonObject* cur = 0;
1289 //      unsigned long res_idx = 0;
1290 //
1291 //      while((cur = jsonObjectGetIndex( obj, res_idx++ ) )) {
1292 //              // We used to discard based on perms here, but now that's
1293 //              // inside doFieldmapperSearch()
1294 //              osrfAppRespond( ctx, cur );
1295 //      }
1296
1297         jsonObjectFree( obj );
1298
1299         osrfAppRespondComplete( ctx, NULL );
1300         return 0;
1301 }
1302
1303 /**
1304         @brief Implement the "id_list" method.
1305         @param ctx Pointer to the method context.
1306         @param err Pointer through which to return an error code.
1307         @return Zero if successful, or -1 if not.
1308
1309         Method parameters:
1310         - authkey (PCRUD only)
1311         - WHERE clause, as jsonObject
1312         - Other SQL clause(s), as a JSON_HASH: joins, LIMIT, etc.
1313
1314         Return to client: The primary key values for all rows of the relevant class that
1315         satisfy a specified WHERE clause.
1316
1317         This method relies on the assumption that every class has a primary key consisting of
1318         a single column.
1319 */
1320 int doIdList( osrfMethodContext* ctx ) {
1321         if( osrfMethodVerifyContext( ctx )) {
1322                 osrfLogError( OSRF_LOG_MARK, "Invalid method context" );
1323                 return -1;
1324         }
1325
1326         if( enforce_pcrud )
1327                 timeout_needs_resetting = 1;
1328
1329         jsonObject* where_clause;
1330         jsonObject* rest_of_query;
1331
1332         // We use the where clause without change.  But we need to massage the rest of the
1333         // query, so we work with a copy of it instead of modifying the original.
1334
1335         if( enforce_pcrud ) {
1336                 where_clause  = jsonObjectGetIndex( ctx->params, 1 );
1337                 rest_of_query = jsonObjectClone( jsonObjectGetIndex( ctx->params, 2 ) );
1338         } else {
1339                 where_clause  = jsonObjectGetIndex( ctx->params, 0 );
1340                 rest_of_query = jsonObjectClone( jsonObjectGetIndex( ctx->params, 1 ) );
1341         }
1342
1343         if( !where_clause ) {
1344                 osrfLogError( OSRF_LOG_MARK, "No WHERE clause parameter supplied" );
1345                 return -1;
1346         }
1347
1348         // Eliminate certain SQL clauses, if present.
1349         if( rest_of_query ) {
1350                 jsonObjectRemoveKey( rest_of_query, "select" );
1351                 jsonObjectRemoveKey( rest_of_query, "no_i18n" );
1352                 jsonObjectRemoveKey( rest_of_query, "flesh" );
1353                 jsonObjectRemoveKey( rest_of_query, "flesh_fields" );
1354         } else {
1355                 rest_of_query = jsonNewObjectType( JSON_HASH );
1356         }
1357
1358         jsonObjectSetKey( rest_of_query, "no_i18n", jsonNewBoolObject( 1 ) );
1359
1360         // Get the class metadata
1361         osrfHash* method_meta = (osrfHash*) ctx->method->userData;
1362         osrfHash* class_meta = osrfHashGet( method_meta, "class" );
1363
1364         // Build a SELECT list containing just the primary key,
1365         // i.e. like { "classname":["keyname"] }
1366         jsonObject* col_list_obj = jsonNewObjectType( JSON_ARRAY );
1367
1368         // Load array with name of primary key
1369         jsonObjectPush( col_list_obj, jsonNewObject( osrfHashGet( class_meta, "primarykey" ) ) );
1370         jsonObject* select_clause = jsonNewObjectType( JSON_HASH );
1371         jsonObjectSetKey( select_clause, osrfHashGet( class_meta, "classname" ), col_list_obj );
1372
1373         jsonObjectSetKey( rest_of_query, "select", select_clause );
1374
1375         // Do the query
1376         int err = 0;
1377         jsonObject* obj =
1378                 doFieldmapperSearch( ctx, class_meta, where_clause, rest_of_query, &err );
1379
1380         jsonObjectFree( rest_of_query );
1381         if( err ) {
1382                 osrfAppRespondComplete( ctx, NULL );
1383                 return -1;
1384         }
1385
1386         // Return each primary key value to the client
1387         jsonObject* cur;
1388         unsigned long res_idx = 0;
1389         while((cur = jsonObjectGetIndex( obj, res_idx++ ) )) {
1390                 // We used to discard based on perms here, but now that's
1391                 // inside doFieldmapperSearch()
1392                 osrfAppRespond( ctx,
1393                         oilsFMGetObject( cur, osrfHashGet( class_meta, "primarykey" ) ) );
1394         }
1395
1396         jsonObjectFree( obj );
1397         osrfAppRespondComplete( ctx, NULL );
1398         return 0;
1399 }
1400
1401 /**
1402         @brief Verify that we have a valid class reference.
1403         @param ctx Pointer to the method context.
1404         @param param Pointer to the method parameters.
1405         @return 1 if the class reference is valid, or zero if it isn't.
1406
1407         The class of the method params must match the class to which the method id devoted.
1408         For PCRUD there are additional restrictions.
1409 */
1410 static int verifyObjectClass ( osrfMethodContext* ctx, const jsonObject* param ) {
1411
1412         osrfHash* method_meta = (osrfHash*) ctx->method->userData;
1413         osrfHash* class = osrfHashGet( method_meta, "class" );
1414
1415         // Compare the method's class to the parameters' class
1416         if( !param->classname || (strcmp( osrfHashGet(class, "classname"), param->classname ))) {
1417
1418                 // Oops -- they don't match.  Complain.
1419                 growing_buffer* msg = buffer_init( 128 );
1420                 buffer_fadd(
1421                         msg,
1422                         "%s: %s method for type %s was passed a %s",
1423                         modulename,
1424                         osrfHashGet( method_meta, "methodtype" ),
1425                         osrfHashGet( class, "classname" ),
1426                         param->classname ? param->classname : "(null)"
1427                 );
1428
1429                 char* m = buffer_release( msg );
1430                 osrfAppSessionStatus( ctx->session, OSRF_STATUS_BADREQUEST, "osrfMethodException",
1431                                 ctx->request, m );
1432                 free( m );
1433
1434                 return 0;
1435         }
1436
1437         if( enforce_pcrud )
1438                 return verifyObjectPCRUD( ctx, class, param, 1 );
1439         else
1440                 return 1;
1441 }
1442
1443 /**
1444         @brief (PCRUD only) Verify that the user is properly logged in.
1445         @param ctx Pointer to the method context.
1446         @return If the user is logged in, a pointer to the user object from the authentication
1447         server; otherwise NULL.
1448 */
1449 static const jsonObject* verifyUserPCRUD( osrfMethodContext* ctx ) {
1450         return verifyUserPCRUDfull( ctx, 0 );
1451 }
1452
1453 static const jsonObject* verifyUserPCRUDfull( osrfMethodContext* ctx, int anon_ok ) {
1454
1455         // Get the authkey (the first method parameter)
1456         const char* auth = jsonObjectGetString( jsonObjectGetIndex( ctx->params, 0 ) );
1457
1458         jsonObject* user = NULL;
1459
1460         // If we are /not/ in anonymous mode
1461         if( strcmp( "ANONYMOUS", auth ) ) {
1462                 // See if we have the same authkey, and a user object,
1463                 // locally cached from a previous call
1464                 const char* cached_authkey = getAuthkey( ctx );
1465                 if( cached_authkey && !strcmp( cached_authkey, auth ) ) {
1466                         const jsonObject* cached_user = getUserLogin( ctx );
1467                         if( cached_user )
1468                                 return cached_user;
1469                 }
1470
1471                 // We have no matching authentication data in the cache.  Authenticate from scratch.
1472                 jsonObject* auth_object = jsonNewObject( auth );
1473
1474                 // Fetch the user object from the authentication server
1475                 user = oilsUtilsQuickReq( "open-ils.auth", "open-ils.auth.session.retrieve", auth_object );
1476                 jsonObjectFree( auth_object );
1477
1478                 if( !user->classname || strcmp(user->classname, "au" )) {
1479
1480                         growing_buffer* msg = buffer_init( 128 );
1481                         buffer_fadd(
1482                                 msg,
1483                                 "%s: permacrud received a bad auth token: %s",
1484                                 modulename,
1485                                 auth
1486                         );
1487
1488                         char* m = buffer_release( msg );
1489                         osrfAppSessionStatus( ctx->session, OSRF_STATUS_UNAUTHORIZED, "osrfMethodException",
1490                                         ctx->request, m );
1491
1492                         free( m );
1493                         jsonObjectFree( user );
1494                         user = NULL;
1495                 } else if( writeAuditInfo( ctx, oilsFMGetStringConst( user, "id" ), oilsFMGetStringConst( user, "wsid" ) ) ) {
1496                         // Failed to set audit information - But note that write_audit_info already set error information.
1497                         jsonObjectFree( user );
1498                         user = NULL;
1499                 }
1500
1501
1502         } else if ( anon_ok ) { // we /are/ (attempting to be) anonymous
1503                 user = jsonNewObjectType(JSON_ARRAY);
1504                 jsonObjectSetClass( user, "aou" );
1505                 oilsFMSetString(user, "id", "-1");
1506         }
1507
1508         setUserLogin( ctx, user );
1509         setAuthkey( ctx, auth );
1510
1511         // Allow ourselves up to a second before we have to reset the login timeout.
1512         // It would be nice to use some fraction of the timeout interval enforced by the
1513         // authentication server, but that value is not readily available at this point.
1514         // Instead, we use a conservative default interval.
1515         time_next_reset = time( NULL ) + 1;
1516
1517         return user;
1518 }
1519
1520 /**
1521         @brief For PCRUD: Determine whether the current user may access the current row.
1522         @param ctx Pointer to the method context.
1523         @param class Same as ctx->method->userData's item for key "class" except when called in recursive doFieldmapperSearch
1524         @param obj Pointer to the row being potentially accessed.
1525         @return 1 if access is permitted, or 0 if it isn't.
1526
1527         The @a obj parameter points to a JSON_HASH of column values, keyed on column name.
1528 */
1529 static int verifyObjectPCRUD ( osrfMethodContext* ctx, osrfHash *class, const jsonObject* obj, int rs_size ) {
1530
1531         dbhandle = writehandle;
1532
1533         // Figure out what class and method are involved
1534         osrfHash* method_metadata = (osrfHash*) ctx->method->userData;
1535         const char* method_type = osrfHashGet( method_metadata, "methodtype" );
1536
1537         if (!rs_size) {
1538                 int *rs_size_from_hash = osrfHashGetFmt( (osrfHash *) ctx->session->userData, "rs_size_req_%d", ctx->request );
1539                 if (rs_size_from_hash) {
1540                         rs_size = *rs_size_from_hash;
1541                         osrfLogDebug(OSRF_LOG_MARK, "used rs_size from request-scoped hash: %d", rs_size);
1542                 }
1543         }
1544
1545         // Set fetch to 1 in all cases except for inserts, meaning that for local or foreign
1546         // contexts we will do another lookup of the current row, even if we already have a
1547         // previously fetched row image, because the row image in hand may not include the
1548         // foreign key(s) that we need.
1549
1550         // This is a quick fix with a bludgeon.  There are ways to avoid the extra lookup,
1551         // but they aren't implemented yet.
1552
1553         int fetch = 0;
1554         if( *method_type == 's' || *method_type == 'i' ) {
1555                 method_type = "retrieve"; // search and id_list are equivalent to retrieve for this
1556                 fetch = 1;
1557         } else if( *method_type == 'u' || *method_type == 'd' ) {
1558                 fetch = 1; // MUST go to the db for the object for update and delete
1559         }
1560
1561         // In retrieve or search ONLY we allow anon. Later perm checks will fail as they should,
1562         // in the face of a fake user but required permissions.
1563         int anon_ok = 0;
1564         if( *method_type == 'r' )
1565                 anon_ok = 1;
1566
1567         // Get the appropriate permacrud entry from the IDL, depending on method type
1568         osrfHash* pcrud = osrfHashGet( osrfHashGet( class, "permacrud" ), method_type );
1569         if( !pcrud ) {
1570                 // No permacrud for this method type on this class
1571
1572                 growing_buffer* msg = buffer_init( 128 );
1573                 buffer_fadd(
1574                         msg,
1575                         "%s: %s on class %s has no permacrud IDL entry",
1576                         modulename,
1577                         osrfHashGet( method_metadata, "methodtype" ),
1578                         osrfHashGet( class, "classname" )
1579                 );
1580
1581                 char* m = buffer_release( msg );
1582                 osrfAppSessionStatus( ctx->session, OSRF_STATUS_FORBIDDEN,
1583                                 "osrfMethodException", ctx->request, m );
1584
1585                 free( m );
1586
1587                 return 0;
1588         }
1589
1590         // Get the user id, and make sure the user is logged in
1591         const jsonObject* user = verifyUserPCRUDfull( ctx, anon_ok );
1592         if( !user )
1593                 return 0;    // Not logged in or anon?  No access.
1594
1595         int userid = atoi( oilsFMGetStringConst( user, "id" ) );
1596
1597         // Get a list of permissions from the permacrud entry.
1598         osrfStringArray* permission = osrfHashGet( pcrud, "permission" );
1599         if( permission->size == 0 ) {
1600                 osrfLogDebug(
1601                         OSRF_LOG_MARK,
1602                         "No permissions required for this action (class %s), passing through",
1603                         osrfHashGet(class, "classname")
1604                 );
1605                 return 1;
1606         }
1607
1608         // But, if there are perms and the user is anonymous ... FAIL
1609         if ( -1 == userid )
1610                 return 0;
1611
1612         // Build a list of org units that own the row.  This is fairly convoluted because there
1613         // are several different ways that an org unit may own the row, as defined by the
1614         // permacrud entry.
1615
1616         // Local context means that the row includes a foreign key pointing to actor.org_unit,
1617         // identifying an owning org_unit..
1618         osrfStringArray* local_context = osrfHashGet( pcrud, "local_context" );
1619
1620         // Foreign context adds a layer of indirection.  The row points to some other row that
1621         // an org unit may own.  The "jump" attribute, if present, adds another layer of
1622         // indirection.
1623         osrfHash* foreign_context = osrfHashGet( pcrud, "foreign_context" );
1624
1625         // The following string array stores the list of org units.  (We don't have a thingie
1626         // for storing lists of integers, so we fake it with a list of strings.)
1627         osrfStringArray* context_org_array = osrfNewStringArray( 1 );
1628
1629         const char* context_org = NULL;
1630     const char* pkey = NULL;
1631     jsonObject *param = NULL;
1632         const char* perm = NULL;
1633         int OK = 0;
1634         int i = 0;
1635         int err = 0;
1636         const char* pkey_value = NULL;
1637         if( str_is_true( osrfHashGet(pcrud, "global_required") ) ) {
1638                 // If the global_required attribute is present and true, then the only owning
1639                 // org unit is the root org unit, i.e. the one with no parent.
1640                 osrfLogDebug( OSRF_LOG_MARK,
1641                                 "global-level permissions required, fetching top of the org tree" );
1642
1643                 // no need to check perms for org tree root retrieval
1644                 osrfHashSet((osrfHash*) ctx->session->userData, "1", "inside_verify");
1645                 // check for perm at top of org tree
1646                 const char* org_tree_root_id = org_tree_root( ctx );
1647                 osrfHashSet((osrfHash*) ctx->session->userData, "0", "inside_verify");
1648
1649                 if( org_tree_root_id ) {
1650                         osrfStringArrayAdd( context_org_array, org_tree_root_id );
1651                         osrfLogDebug( OSRF_LOG_MARK, "top of the org tree is %s", org_tree_root_id );
1652                 } else  {
1653                         osrfStringArrayFree( context_org_array );
1654                         return 0;
1655                 }
1656
1657         } else {
1658                 // If the global_required attribute is absent or false, then we look for
1659                 // local and/or foreign context.  In order to find the relevant foreign
1660                 // keys, we must either read the relevant row from the database, or look at
1661                 // the image of the row that we already have in memory.
1662
1663                 // Even if we have an image of the row in memory, that image may not include the
1664                 // foreign key column(s) that we need.  So whenever possible, we do a fresh read
1665                 // of the row to make sure that we have what we need.
1666
1667             osrfLogDebug( OSRF_LOG_MARK, "global-level permissions not required, "
1668                                 "fetching context org ids" );
1669
1670         pkey = osrfHashGet( class, "primarykey" );
1671
1672                 if( !pkey ) {
1673                         // There is no primary key, so we can't do a fresh lookup.  Use the row
1674                         // image that we already have.  If it doesn't have everything we need, too bad.
1675                         fetch = 0;
1676                         param = jsonObjectClone( obj );
1677                         osrfLogDebug( OSRF_LOG_MARK, "No primary key; using clone of object" );
1678                 } else if( obj->classname ) {
1679                         pkey_value = oilsFMGetStringConst( obj, pkey );
1680                         if( !fetch )
1681                                 param = jsonObjectClone( obj );
1682                         osrfLogDebug( OSRF_LOG_MARK, "Object supplied, using primary key value of %s",
1683                                 pkey_value );
1684                 } else {
1685                         pkey_value = jsonObjectGetString( obj );
1686                         fetch = 1;
1687                         osrfLogDebug( OSRF_LOG_MARK, "Object not supplied, using primary key value "
1688                                 "of %s and retrieving from the database", pkey_value );
1689                 }
1690
1691                 if( fetch ) {
1692                         // Fetch the row so that we can look at the foreign key(s)
1693                         osrfHashSet((osrfHash*) ctx->session->userData, "1", "inside_verify");
1694                         jsonObject* _tmp_params = single_hash( pkey, pkey_value );
1695                         jsonObject* _list = doFieldmapperSearch( ctx, class, _tmp_params, NULL, &err );
1696                         jsonObjectFree( _tmp_params );
1697                         osrfHashSet((osrfHash*) ctx->session->userData, "0", "inside_verify");
1698
1699                         param = jsonObjectExtractIndex( _list, 0 );
1700                         jsonObjectFree( _list );
1701
1702             fetch = 0;
1703                 }
1704
1705                 if( !param ) {
1706                         // The row doesn't exist.  Complain, and deny access.
1707                         osrfLogDebug( OSRF_LOG_MARK,
1708                                         "Object not found in the database with primary key %s of %s",
1709                                         pkey, pkey_value );
1710
1711                         growing_buffer* msg = buffer_init( 128 );
1712                         buffer_fadd(
1713                                 msg,
1714                                 "%s: no object found with primary key %s of %s",
1715                                 modulename,
1716                                 pkey,
1717                                 pkey_value
1718                         );
1719
1720                         char* m = buffer_release( msg );
1721                         osrfAppSessionStatus(
1722                                 ctx->session,
1723                                 OSRF_STATUS_INTERNALSERVERERROR,
1724                                 "osrfMethodException",
1725                                 ctx->request,
1726                                 m
1727                         );
1728
1729                         free( m );
1730                         return 0;
1731                 }
1732
1733                 if( local_context && local_context->size > 0 ) {
1734                         // The IDL provides a list of column names for the foreign keys denoting
1735                         // local context, i.e. columns identifying owing org units directly.  Look up
1736                         // the value of each one, and if it isn't null, add it to the list of org units.
1737                         osrfLogDebug( OSRF_LOG_MARK, "%d class-local context field(s) specified",
1738                                 local_context->size );
1739                         int i = 0;
1740                         const char* lcontext = NULL;
1741                         while ( (lcontext = osrfStringArrayGetString(local_context, i++)) ) {
1742                                 const char* fkey_value = oilsFMGetStringConst( param, lcontext );
1743                                 if( fkey_value ) {    // if not null
1744                                         osrfStringArrayAdd( context_org_array, fkey_value );
1745                                         osrfLogDebug(
1746                                                 OSRF_LOG_MARK,
1747                                                 "adding class-local field %s (value: %s) to the context org list",
1748                                                 lcontext,
1749                                                 osrfStringArrayGetString( context_org_array, context_org_array->size - 1 )
1750                                         );
1751                                 }
1752                         }
1753                 }
1754
1755                 if( foreign_context ) {
1756                         unsigned long class_count = osrfHashGetCount( foreign_context );
1757                         osrfLogDebug( OSRF_LOG_MARK, "%d foreign context classes(s) specified", class_count );
1758
1759                         if( class_count > 0 ) {
1760
1761                                 // The IDL provides a list of foreign key columns pointing to rows that
1762                                 // an org unit may own.  Follow each link, identify the owning org unit,
1763                                 // and add it to the list.
1764                                 osrfHash* fcontext = NULL;
1765                                 osrfHashIterator* class_itr = osrfNewHashIterator( foreign_context );
1766                                 while( (fcontext = osrfHashIteratorNext( class_itr )) ) {
1767                                         // For each class to which a foreign key points:
1768                                         const char* class_name = osrfHashIteratorKey( class_itr );
1769                                         osrfHash* fcontext = osrfHashGet( foreign_context, class_name );
1770
1771                                         osrfLogDebug(
1772                                                 OSRF_LOG_MARK,
1773                                                 "%d foreign context fields(s) specified for class %s",
1774                                                 ((osrfStringArray*)osrfHashGet(fcontext,"context"))->size,
1775                                                 class_name
1776                                         );
1777
1778                                         // Get the name of the key field in the foreign table
1779                                         const char* foreign_pkey = osrfHashGet( fcontext, "field" );
1780
1781                                         // Get the value of the foreign key pointing to the foreign table
1782                                         char* foreign_pkey_value =
1783                                                         oilsFMGetString( param, osrfHashGet( fcontext, "fkey" ));
1784                                         if( !foreign_pkey_value )
1785                                                 continue;    // Foreign key value is null; skip it
1786
1787                                         // Look up the row to which the foreign key points
1788                                         jsonObject* _tmp_params = single_hash( foreign_pkey, foreign_pkey_value );
1789
1790                                         osrfHashSet((osrfHash*) ctx->session->userData, "1", "inside_verify");
1791                                         jsonObject* _list = doFieldmapperSearch(
1792                                                 ctx, osrfHashGet( oilsIDL(), class_name ), _tmp_params, NULL, &err );
1793                                         osrfHashSet((osrfHash*) ctx->session->userData, "0", "inside_verify");
1794
1795                                         jsonObject* _fparam = NULL;
1796                                         if( _list && JSON_ARRAY == _list->type && _list->size > 0 )
1797                                                 _fparam = jsonObjectExtractIndex( _list, 0 );
1798
1799                                         jsonObjectFree( _tmp_params );
1800                                         jsonObjectFree( _list );
1801
1802                                         // At this point _fparam either points to the row identified by the
1803                                         // foreign key, or it's NULL (no such row found).
1804
1805                                         osrfStringArray* jump_list = osrfHashGet( fcontext, "jump" );
1806
1807                                         const char* bad_class = NULL;  // For noting failed lookups
1808                                         if( ! _fparam )
1809                                                 bad_class = class_name;    // Referenced row not found
1810                                         else if( jump_list ) {
1811                                                 // Follow a chain of rows, linked by foreign keys, to find an owner
1812                                                 const char* flink = NULL;
1813                                                 int k = 0;
1814                                                 while ( (flink = osrfStringArrayGetString(jump_list, k++)) && _fparam ) {
1815                                                         // For each entry in the jump list.  Each entry (i.e. flink) is
1816                                                         // the name of a foreign key column in the current row.
1817
1818                                                         // From the IDL, get the linkage information for the next jump
1819                                                         osrfHash* foreign_link_hash =
1820                                                                         oilsIDLFindPath( "/%s/links/%s", _fparam->classname, flink );
1821
1822                                                         // Get the class metadata for the class
1823                                                         // to which the foreign key points
1824                                                         osrfHash* foreign_class_meta = osrfHashGet( oilsIDL(),
1825                                                                         osrfHashGet( foreign_link_hash, "class" ));
1826
1827                                                         // Get the name of the referenced key of that class
1828                                                         foreign_pkey = osrfHashGet( foreign_link_hash, "key" );
1829
1830                                                         // Get the value of the foreign key pointing to that class
1831                                                         free( foreign_pkey_value );
1832                                                         foreign_pkey_value = oilsFMGetString( _fparam, flink );
1833                                                         if( !foreign_pkey_value )
1834                                                                 break;    // Foreign key is null; quit looking
1835
1836                                                         // Build a WHERE clause for the lookup
1837                                                         _tmp_params = single_hash( foreign_pkey, foreign_pkey_value );
1838
1839                                                         // Do the lookup
1840                                                         osrfHashSet((osrfHash*) ctx->session->userData, "1", "inside_verify");
1841                                                         _list = doFieldmapperSearch( ctx, foreign_class_meta,
1842                                                                         _tmp_params, NULL, &err );
1843                                                         osrfHashSet((osrfHash*) ctx->session->userData, "0", "inside_verify");
1844
1845                                                         // Get the resulting row
1846                                                         jsonObjectFree( _fparam );
1847                                                         if( _list && JSON_ARRAY == _list->type && _list->size > 0 )
1848                                                                 _fparam = jsonObjectExtractIndex( _list, 0 );
1849                                                         else {
1850                                                                 // Referenced row not found
1851                                                                 _fparam = NULL;
1852                                                                 bad_class = osrfHashGet( foreign_link_hash, "class" );
1853                                                         }
1854
1855                                                         jsonObjectFree( _tmp_params );
1856                                                         jsonObjectFree( _list );
1857                                                 }
1858                                         }
1859
1860                                         if( bad_class ) {
1861
1862                                                 // We had a foreign key pointing to such-and-such a row, but then
1863                                                 // we couldn't fetch that row.  The data in the database are in an
1864                                                 // inconsistent state; the database itself may even be corrupted.
1865                                                 growing_buffer* msg = buffer_init( 128 );
1866                                                 buffer_fadd(
1867                                                         msg,
1868                                                         "%s: no object of class %s found with primary key %s of %s",
1869                                                         modulename,
1870                                                         bad_class,
1871                                                         foreign_pkey,
1872                                                         foreign_pkey_value ? foreign_pkey_value : "(null)"
1873                                                 );
1874
1875                                                 char* m = buffer_release( msg );
1876                                                 osrfAppSessionStatus(
1877                                                         ctx->session,
1878                                                         OSRF_STATUS_INTERNALSERVERERROR,
1879                                                         "osrfMethodException",
1880                                                         ctx->request,
1881                                                         m
1882                                                 );
1883
1884                                                 free( m );
1885                                                 osrfHashIteratorFree( class_itr );
1886                                                 free( foreign_pkey_value );
1887                                                 jsonObjectFree( param );
1888
1889                                                 return 0;
1890                                         }
1891
1892                                         free( foreign_pkey_value );
1893
1894                                         if( _fparam ) {
1895                                                 // Examine each context column of the foreign row,
1896                                                 // and add its value to the list of org units.
1897                                                 int j = 0;
1898                                                 const char* foreign_field = NULL;
1899                                                 osrfStringArray* ctx_array = osrfHashGet( fcontext, "context" );
1900                                                 while ( (foreign_field = osrfStringArrayGetString( ctx_array, j++ )) ) {
1901                                                         osrfStringArrayAdd( context_org_array,
1902                                                                 oilsFMGetStringConst( _fparam, foreign_field ));
1903                                                         osrfLogDebug( OSRF_LOG_MARK,
1904                                                                 "adding foreign class %s field %s (value: %s) "
1905                                                                         "to the context org list",
1906                                                                 class_name,
1907                                                                 foreign_field,
1908                                                                 osrfStringArrayGetString(
1909                                                                         context_org_array, context_org_array->size - 1 )
1910                                                         );
1911                                                 }
1912
1913                                                 jsonObjectFree( _fparam );
1914                                         }
1915                                 }
1916
1917                                 osrfHashIteratorFree( class_itr );
1918                         }
1919                 }
1920         }
1921
1922     // If there is an owning_user attached to the action, we allow that user and users with
1923     // object perms on the object. CREATE can't use this. We only do this when we're not
1924     // ignoring object perms.
1925     char* owning_user_field = osrfHashGet( pcrud, "owning_user" );
1926     if (
1927         *method_type != 'c' &&
1928         (!str_is_true( osrfHashGet(pcrud, "ignore_object_perms") ) || // Always honor
1929         owning_user_field)
1930     ) {
1931         if (owning_user_field) { // see if we can short-cut by comparing the owner to the requestor
1932
1933             if (!param) { // We didn't get it during the context lookup
1934                             pkey = osrfHashGet( class, "primarykey" );
1935
1936                                 if( !pkey  ) {
1937                                         // There is no primary key, so we can't do a fresh lookup.  Use the row
1938                                         // image that we already have.  If it doesn't have everything we need, too bad.
1939                                         fetch = 0;
1940                                         param = jsonObjectClone( obj );
1941                                         osrfLogDebug( OSRF_LOG_MARK, "No primary key; using clone of object" );
1942                                 } else if( obj->classname ) {
1943                                         pkey_value = oilsFMGetStringConst( obj, pkey );
1944                                         if( !fetch )
1945                                                 param = jsonObjectClone( obj );
1946                                         osrfLogDebug( OSRF_LOG_MARK, "Object supplied, using primary key value of %s",
1947                                                 pkey_value );
1948                                 } else {
1949                                         pkey_value = jsonObjectGetString( obj );
1950                                         fetch = 1;
1951                                         osrfLogDebug( OSRF_LOG_MARK, "Object not supplied, using primary key value "
1952                                                 "of %s and retrieving from the database", pkey_value );
1953                                 }
1954
1955                                 if( fetch ) {
1956                                         // Fetch the row so that we can look at the foreign key(s)
1957                                         osrfHashSet((osrfHash*) ctx->session->userData, "1", "inside_verify");
1958                                         jsonObject* _tmp_params = single_hash( pkey, pkey_value );
1959                                         jsonObject* _list = doFieldmapperSearch( ctx, class, _tmp_params, NULL, &err );
1960                                         jsonObjectFree( _tmp_params );
1961                                         osrfHashSet((osrfHash*) ctx->session->userData, "0", "inside_verify");
1962
1963                                         param = jsonObjectExtractIndex( _list, 0 );
1964                                         jsonObjectFree( _list );
1965                                 }
1966             }
1967
1968                         if( !param ) {
1969                                 // The row doesn't exist.  Complain, and deny access.
1970                                 osrfLogDebug( OSRF_LOG_MARK,
1971                                                 "Object not found in the database with primary key %s of %s",
1972                                                 pkey, pkey_value );
1973
1974                                 growing_buffer* msg = buffer_init( 128 );
1975                                 buffer_fadd(
1976                                         msg,
1977                                         "%s: no object found with primary key %s of %s",
1978                                         modulename,
1979                                         pkey,
1980                                         pkey_value
1981                                 );
1982
1983                                 char* m = buffer_release( msg );
1984                                 osrfAppSessionStatus(
1985                                         ctx->session,
1986                                         OSRF_STATUS_INTERNALSERVERERROR,
1987                                         "osrfMethodException",
1988                                         ctx->request,
1989                                         m
1990                                 );
1991
1992                                 free( m );
1993                                 return 0;
1994                         } else {
1995
1996                 int ownerid = atoi( oilsFMGetStringConst( param, owning_user_field ) );
1997
1998                 // Allow the owner to do whatever
1999                 if (ownerid == userid)
2000                     OK = 1;
2001                         }
2002                 }
2003
2004         i = 0;
2005         while(  !OK &&
2006                                 (perm = osrfStringArrayGetString(permission, i++)) &&
2007                                 !str_is_true( osrfHashGet(pcrud, "ignore_object_perms"))
2008                 ) {
2009                 dbi_result result;
2010
2011                         osrfLogDebug(
2012                                 OSRF_LOG_MARK,
2013                                 "Checking object permission [%s] for user %d "
2014                                                 "on object %s (class %s)",
2015                                 perm,
2016                                 userid,
2017                                 pkey_value,
2018                                 osrfHashGet( class, "classname" )
2019                         );
2020
2021                         result = dbi_conn_queryf(
2022                                 writehandle,
2023                                 "SELECT permission.usr_has_object_perm(%d, '%s', '%s', '%s') AS has_perm;",
2024                                 userid,
2025                                 perm,
2026                                 osrfHashGet( class, "classname" ),
2027                                 pkey_value
2028                         );
2029
2030                         if( result ) {
2031                                 osrfLogDebug(
2032                                         OSRF_LOG_MARK,
2033                                         "Received a result for object permission [%s] "
2034                                                         "for user %d on object %s (class %s)",
2035                                         perm,
2036                                         userid,
2037                                         pkey_value,
2038                                         osrfHashGet( class, "classname" )
2039                                 );
2040
2041                                 if( dbi_result_first_row( result )) {
2042                                         jsonObject* return_val = oilsMakeJSONFromResult( result );
2043                                         const char* has_perm = jsonObjectGetString(
2044                                                         jsonObjectGetKeyConst( return_val, "has_perm" ));
2045
2046                                         osrfLogDebug(
2047                                                 OSRF_LOG_MARK,
2048                                                 "Status of object permission [%s] for user %d "
2049                                                                 "on object %s (class %s) is %s",
2050                                                 perm,
2051                                                 userid,
2052                                                 pkey_value,
2053                                                 osrfHashGet(class, "classname"),
2054                                                 has_perm
2055                                         );
2056
2057                                         if( *has_perm == 't' )
2058                                                 OK = 1;
2059                                         jsonObjectFree( return_val );
2060                                 }
2061
2062                                 dbi_result_free( result );
2063                                 if( OK )
2064                     break;
2065                         } else {
2066                                 const char* msg;
2067                                 int errnum = dbi_conn_error( writehandle, &msg );
2068                                 osrfLogWarning( OSRF_LOG_MARK,
2069                                         "Unable to call check object permissions: %d, %s",
2070                                         errnum, msg ? msg : "(No description available)" );
2071                                 if( !oilsIsDBConnected( writehandle ))
2072                                         osrfAppSessionPanic( ctx->session );
2073                         }
2074         }
2075     }
2076
2077         // For every combination of permission and context org unit: call a stored procedure
2078         // to determine if the user has this permission in the context of this org unit.
2079         // If the answer is yes at any point, then we're done, and the user has permission.
2080         // In other words permissions are additive.
2081         i = 0;
2082         while( !OK && (perm = osrfStringArrayGetString(permission, i++)) ) {
2083                 dbi_result result;
2084
2085         osrfStringArray* pcache = NULL;
2086         if (rs_size > perm_at_threshold) { // grab and cache locations of user perms
2087                         pcache = getPermLocationCache(ctx, perm);
2088
2089                         if (!pcache) {
2090                         pcache = osrfNewStringArray(0);
2091
2092                                 result = dbi_conn_queryf(
2093                                         writehandle,
2094                                         "SELECT permission.usr_has_perm_at_all(%d, '%s') AS at;",
2095                                         userid,
2096                                         perm
2097                                 );
2098
2099                                 if( result ) {
2100                                         osrfLogDebug(
2101                                                 OSRF_LOG_MARK,
2102                                                 "Received a result for permission [%s] for user %d",
2103                                                 perm,
2104                                                 userid
2105                                         );
2106
2107                                         if( dbi_result_first_row( result )) {
2108                             do {
2109                                                 jsonObject* return_val = oilsMakeJSONFromResult( result );
2110                                                 osrfStringArrayAdd( pcache, jsonObjectGetString( jsonObjectGetKeyConst( return_val, "at" ) ) );
2111                                 jsonObjectFree( return_val );
2112                                             } while( dbi_result_next_row( result ));
2113
2114                                                 setPermLocationCache(ctx, perm, pcache);
2115                                         }
2116
2117                                         dbi_result_free( result );
2118                     }
2119                         }
2120         }
2121
2122                 int j = 0;
2123                 while( (context_org = osrfStringArrayGetString( context_org_array, j++ )) ) {
2124
2125             if (rs_size > perm_at_threshold) {
2126                 if (osrfStringArrayContains( pcache, context_org )) {
2127                     OK = 1;
2128                     break;
2129                 }
2130             }
2131
2132                         if(
2133                 pkey_value &&
2134                 !str_is_true( osrfHashGet(pcrud, "ignore_object_perms") ) && // Always honor
2135                 (
2136                     !str_is_true( osrfHashGet(pcrud, "global_required") ) ||
2137                     osrfHashGet(pcrud, "owning_user")
2138                 )
2139             ) {
2140                                 osrfLogDebug(
2141                                         OSRF_LOG_MARK,
2142                                         "Checking object permission [%s] for user %d "
2143                                                         "on object %s (class %s) at org %d",
2144                                         perm,
2145                                         userid,
2146                                         pkey_value,
2147                                         osrfHashGet( class, "classname" ),
2148                                         atoi( context_org )
2149                                 );
2150
2151                                 result = dbi_conn_queryf(
2152                                         writehandle,
2153                                         "SELECT permission.usr_has_object_perm(%d, '%s', '%s', '%s', %d) AS has_perm;",
2154                                         userid,
2155                                         perm,
2156                                         osrfHashGet( class, "classname" ),
2157                                         pkey_value,
2158                                         atoi( context_org )
2159                                 );
2160
2161                                 if( result ) {
2162                                         osrfLogDebug(
2163                                                 OSRF_LOG_MARK,
2164                                                 "Received a result for object permission [%s] "
2165                                                                 "for user %d on object %s (class %s) at org %d",
2166                                                 perm,
2167                                                 userid,
2168                                                 pkey_value,
2169                                                 osrfHashGet( class, "classname" ),
2170                                                 atoi( context_org )
2171                                         );
2172
2173                                         if( dbi_result_first_row( result )) {
2174                                                 jsonObject* return_val = oilsMakeJSONFromResult( result );
2175                                                 const char* has_perm = jsonObjectGetString(
2176                                                                 jsonObjectGetKeyConst( return_val, "has_perm" ));
2177
2178                                                 osrfLogDebug(
2179                                                         OSRF_LOG_MARK,
2180                                                         "Status of object permission [%s] for user %d "
2181                                                                         "on object %s (class %s) at org %d is %s",
2182                                                         perm,
2183                                                         userid,
2184                                                         pkey_value,
2185                                                         osrfHashGet(class, "classname"),
2186                                                         atoi(context_org),
2187                                                         has_perm
2188                                                 );
2189
2190                                                 if( *has_perm == 't' )
2191                                                         OK = 1;
2192                                                 jsonObjectFree( return_val );
2193                                         }
2194
2195                                         dbi_result_free( result );
2196                                         if( OK )
2197                                                 break;
2198                                 } else {
2199                                         const char* msg;
2200                                         int errnum = dbi_conn_error( writehandle, &msg );
2201                                         osrfLogWarning( OSRF_LOG_MARK,
2202                                                 "Unable to call check object permissions: %d, %s",
2203                                                 errnum, msg ? msg : "(No description available)" );
2204                                         if( !oilsIsDBConnected( writehandle ))
2205                                                 osrfAppSessionPanic( ctx->session );
2206                                 }
2207                         }
2208
2209             if (rs_size > perm_at_threshold) break;
2210
2211                         osrfLogDebug( OSRF_LOG_MARK,
2212                                         "Checking non-object permission [%s] for user %d at org %d",
2213                                         perm, userid, atoi(context_org) );
2214                         result = dbi_conn_queryf(
2215                                 writehandle,
2216                                 "SELECT permission.usr_has_perm(%d, '%s', %d) AS has_perm;",
2217                                 userid,
2218                                 perm,
2219                                 atoi( context_org )
2220                         );
2221
2222                         if( result ) {
2223                                 osrfLogDebug( OSRF_LOG_MARK,
2224                                         "Received a result for permission [%s] for user %d at org %d",
2225                                         perm, userid, atoi( context_org ));
2226                                 if( dbi_result_first_row( result )) {
2227                                         jsonObject* return_val = oilsMakeJSONFromResult( result );
2228                                         const char* has_perm = jsonObjectGetString(
2229                                                 jsonObjectGetKeyConst( return_val, "has_perm" ));
2230                                         osrfLogDebug( OSRF_LOG_MARK,
2231                                                 "Status of permission [%s] for user %d at org %d is [%s]",
2232                                                 perm, userid, atoi( context_org ), has_perm );
2233                                         if( *has_perm == 't' )
2234                                                 OK = 1;
2235                                         jsonObjectFree( return_val );
2236                                 }
2237
2238                                 dbi_result_free( result );
2239                                 if( OK )
2240                                         break;
2241                         } else {
2242                                 const char* msg;
2243                                 int errnum = dbi_conn_error( writehandle, &msg );
2244                                 osrfLogWarning( OSRF_LOG_MARK, "Unable to call user object permissions: %d, %s",
2245                                         errnum, msg ? msg : "(No description available)" );
2246                                 if( !oilsIsDBConnected( writehandle ))
2247                                         osrfAppSessionPanic( ctx->session );
2248                         }
2249
2250                 }
2251
2252                 if( OK )
2253                         break;
2254         }
2255
2256         osrfStringArrayFree( context_org_array );
2257
2258         return OK;
2259 }
2260
2261 /**
2262         @brief Look up the root of the org_unit tree.
2263         @param ctx Pointer to the method context.
2264         @return The id of the root org unit, as a character string.
2265
2266         Query actor.org_unit where parent_ou is null, and return the id as a string.
2267
2268         This function assumes that there is only one root org unit, i.e. that we
2269         have a single tree, not a forest.
2270
2271         The calling code is responsible for freeing the returned string.
2272 */
2273 static const char* org_tree_root( osrfMethodContext* ctx ) {
2274
2275         static char cached_root_id[ 32 ] = "";  // extravagantly large buffer
2276         static time_t last_lookup_time = 0;
2277         time_t current_time = time( NULL );
2278
2279         if( cached_root_id[ 0 ] && ( current_time - last_lookup_time < 3600 ) ) {
2280                 // We successfully looked this up less than an hour ago.
2281                 // It's not likely to have changed since then.
2282                 return strdup( cached_root_id );
2283         }
2284         last_lookup_time = current_time;
2285
2286         int err = 0;
2287         jsonObject* where_clause = single_hash( "parent_ou", NULL );
2288         jsonObject* result = doFieldmapperSearch(
2289                 ctx, osrfHashGet( oilsIDL(), "aou" ), where_clause, NULL, &err );
2290         jsonObjectFree( where_clause );
2291
2292         jsonObject* tree_top = jsonObjectGetIndex( result, 0 );
2293
2294         if( !tree_top ) {
2295                 jsonObjectFree( result );
2296
2297                 growing_buffer* msg = buffer_init( 128 );
2298                 OSRF_BUFFER_ADD( msg, modulename );
2299                 OSRF_BUFFER_ADD( msg,
2300                                 ": Internal error, could not find the top of the org tree (parent_ou = NULL)" );
2301
2302                 char* m = buffer_release( msg );
2303                 osrfAppSessionStatus( ctx->session,
2304                                 OSRF_STATUS_INTERNALSERVERERROR, "osrfMethodException", ctx->request, m );
2305                 free( m );
2306
2307                 cached_root_id[ 0 ] = '\0';
2308                 return NULL;
2309         }
2310
2311         const char* root_org_unit_id = oilsFMGetStringConst( tree_top, "id" );
2312         osrfLogDebug( OSRF_LOG_MARK, "Top of the org tree is %s", root_org_unit_id );
2313
2314         strcpy( cached_root_id, root_org_unit_id );
2315         jsonObjectFree( result );
2316         return cached_root_id;
2317 }
2318
2319 /**
2320         @brief Create a JSON_HASH with a single key/value pair.
2321         @param key The key of the key/value pair.
2322         @param value the value of the key/value pair.
2323         @return Pointer to a newly created jsonObject of type JSON_HASH.
2324
2325         The value of the key/value is either a string or (if @a value is NULL) a null.
2326 */
2327 static jsonObject* single_hash( const char* key, const char* value ) {
2328         // Sanity check
2329         if( ! key ) key = "";
2330
2331         jsonObject* hash = jsonNewObjectType( JSON_HASH );
2332         jsonObjectSetKey( hash, key, jsonNewObject( value ) );
2333         return hash;
2334 }
2335
2336
2337 int doCreate( osrfMethodContext* ctx ) {
2338         if(osrfMethodVerifyContext( ctx )) {
2339                 osrfLogError( OSRF_LOG_MARK, "Invalid method context" );
2340                 return -1;
2341         }
2342
2343         if( enforce_pcrud )
2344                 timeout_needs_resetting = 1;
2345
2346         osrfHash* meta = osrfHashGet( (osrfHash*) ctx->method->userData, "class" );
2347         jsonObject* target = NULL;
2348         jsonObject* options = NULL;
2349
2350         if( enforce_pcrud ) {
2351                 target = jsonObjectGetIndex( ctx->params, 1 );
2352                 options = jsonObjectGetIndex( ctx->params, 2 );
2353         } else {
2354                 target = jsonObjectGetIndex( ctx->params, 0 );
2355                 options = jsonObjectGetIndex( ctx->params, 1 );
2356         }
2357
2358         if( !verifyObjectClass( ctx, target )) {
2359                 osrfAppRespondComplete( ctx, NULL );
2360                 return -1;
2361         }
2362
2363         osrfLogDebug( OSRF_LOG_MARK, "Object seems to be of the correct type" );
2364
2365         const char* trans_id = getXactId( ctx );
2366         if( !trans_id ) {
2367                 osrfLogError( OSRF_LOG_MARK, "No active transaction -- required for CREATE" );
2368
2369                 osrfAppSessionStatus(
2370                         ctx->session,
2371                         OSRF_STATUS_BADREQUEST,
2372                         "osrfMethodException",
2373                         ctx->request,
2374                         "No active transaction -- required for CREATE"
2375                 );
2376                 osrfAppRespondComplete( ctx, NULL );
2377                 return -1;
2378         }
2379
2380         // The following test is harmless but redundant.  If a class is
2381         // readonly, we don't register a create method for it.
2382         if( str_is_true( osrfHashGet( meta, "readonly" ) ) ) {
2383                 osrfAppSessionStatus(
2384                         ctx->session,
2385                         OSRF_STATUS_BADREQUEST,
2386                         "osrfMethodException",
2387                         ctx->request,
2388                         "Cannot INSERT readonly class"
2389                 );
2390                 osrfAppRespondComplete( ctx, NULL );
2391                 return -1;
2392         }
2393
2394         // Set the last_xact_id
2395         int index = oilsIDL_ntop( target->classname, "last_xact_id" );
2396         if( index > -1 ) {
2397                 osrfLogDebug(OSRF_LOG_MARK, "Setting last_xact_id to %s on %s at position %d",
2398                         trans_id, target->classname, index);
2399                 jsonObjectSetIndex( target, index, jsonNewObject( trans_id ));
2400         }
2401
2402         osrfLogDebug( OSRF_LOG_MARK, "There is a transaction running..." );
2403
2404         dbhandle = writehandle;
2405
2406         osrfHash* fields = osrfHashGet( meta, "fields" );
2407         char* pkey       = osrfHashGet( meta, "primarykey" );
2408         char* seq        = osrfHashGet( meta, "sequence" );
2409
2410         growing_buffer* table_buf = buffer_init( 128 );
2411         growing_buffer* col_buf   = buffer_init( 128 );
2412         growing_buffer* val_buf   = buffer_init( 128 );
2413
2414         OSRF_BUFFER_ADD( table_buf, "INSERT INTO " );
2415         OSRF_BUFFER_ADD( table_buf, osrfHashGet( meta, "tablename" ));
2416         OSRF_BUFFER_ADD_CHAR( col_buf, '(' );
2417         buffer_add( val_buf,"VALUES (" );
2418
2419
2420         int first = 1;
2421         osrfHash* field = NULL;
2422         osrfHashIterator* field_itr = osrfNewHashIterator( fields );
2423         while( (field = osrfHashIteratorNext( field_itr ) ) ) {
2424
2425                 const char* field_name = osrfHashIteratorKey( field_itr );
2426
2427                 if( str_is_true( osrfHashGet( field, "virtual" ) ) )
2428                         continue;
2429
2430                 const jsonObject* field_object = oilsFMGetObject( target, field_name );
2431
2432                 char* value;
2433                 if( field_object && field_object->classname ) {
2434                         value = oilsFMGetString(
2435                                 field_object,
2436                                 (char*)oilsIDLFindPath( "/%s/primarykey", field_object->classname )
2437                         );
2438                 } else if( field_object && JSON_BOOL == field_object->type ) {
2439                         if( jsonBoolIsTrue( field_object ) )
2440                                 value = strdup( "t" );
2441                         else
2442                                 value = strdup( "f" );
2443                 } else {
2444                         value = jsonObjectToSimpleString( field_object );
2445                 }
2446
2447                 if( first ) {
2448                         first = 0;
2449                 } else {
2450                         OSRF_BUFFER_ADD_CHAR( col_buf, ',' );
2451                         OSRF_BUFFER_ADD_CHAR( val_buf, ',' );
2452                 }
2453
2454                 buffer_add( col_buf, field_name );
2455
2456                 if( !field_object || field_object->type == JSON_NULL ) {
2457                         buffer_add( val_buf, "DEFAULT" );
2458
2459                 } else if( !strcmp( get_primitive( field ), "number" )) {
2460                         const char* numtype = get_datatype( field );
2461                         if( !strcmp( numtype, "INT8" )) {
2462                                 buffer_fadd( val_buf, "%lld", atoll( value ));
2463
2464                         } else if( !strcmp( numtype, "INT" )) {
2465                                 buffer_fadd( val_buf, "%d", atoi( value ));
2466
2467                         } else if( !strcmp( numtype, "NUMERIC" )) {
2468                                 buffer_fadd( val_buf, "%f", atof( value ));
2469                         }
2470                 } else {
2471                         if( dbi_conn_quote_string( writehandle, &value )) {
2472                                 OSRF_BUFFER_ADD( val_buf, value );
2473
2474                         } else {
2475                                 osrfLogError( OSRF_LOG_MARK, "%s: Error quoting string [%s]", modulename, value );
2476                                 osrfAppSessionStatus(
2477                                         ctx->session,
2478                                         OSRF_STATUS_INTERNALSERVERERROR,
2479                                         "osrfMethodException",
2480                                         ctx->request,
2481                                         "Error quoting string -- please see the error log for more details"
2482                                 );
2483                                 free( value );
2484                                 buffer_free( table_buf );
2485                                 buffer_free( col_buf );
2486                                 buffer_free( val_buf );
2487                                 osrfAppRespondComplete( ctx, NULL );
2488                                 return -1;
2489                         }
2490                 }
2491
2492                 free( value );
2493         }
2494
2495         osrfHashIteratorFree( field_itr );
2496
2497         OSRF_BUFFER_ADD_CHAR( col_buf, ')' );
2498         OSRF_BUFFER_ADD_CHAR( val_buf, ')' );
2499
2500         char* table_str = buffer_release( table_buf );
2501         char* col_str   = buffer_release( col_buf );
2502         char* val_str   = buffer_release( val_buf );
2503         growing_buffer* sql = buffer_init( 128 );
2504         buffer_fadd( sql, "%s %s %s;", table_str, col_str, val_str );
2505         free( table_str );
2506         free( col_str );
2507         free( val_str );
2508
2509         char* query = buffer_release( sql );
2510
2511         osrfLogDebug( OSRF_LOG_MARK, "%s: Insert SQL [%s]", modulename, query );
2512
2513         jsonObject* obj = NULL;
2514         int rc = 0;
2515
2516         dbi_result result = dbi_conn_query( writehandle, query );
2517         if( !result ) {
2518                 obj = jsonNewObject( NULL );
2519                 const char* msg;
2520                 int errnum = dbi_conn_error( writehandle, &msg );
2521                 osrfLogError(
2522                         OSRF_LOG_MARK,
2523                         "%s ERROR inserting %s object using query [%s]: %d %s",
2524                         modulename,
2525                         osrfHashGet(meta, "fieldmapper"),
2526                         query,
2527                         errnum,
2528                         msg ? msg : "(No description available)"
2529                 );
2530                 osrfAppSessionStatus(
2531                         ctx->session,
2532                         OSRF_STATUS_INTERNALSERVERERROR,
2533                         "osrfMethodException",
2534                         ctx->request,
2535                         "INSERT error -- please see the error log for more details"
2536                 );
2537                 if( !oilsIsDBConnected( writehandle ))
2538                         osrfAppSessionPanic( ctx->session );
2539                 rc = -1;
2540         } else {
2541                 dbi_result_free( result );
2542
2543                 char* id = oilsFMGetString( target, pkey );
2544                 if( !id ) {
2545                         unsigned long long new_id = dbi_conn_sequence_last( writehandle, seq );
2546                         growing_buffer* _id = buffer_init( 10 );
2547                         buffer_fadd( _id, "%lld", new_id );
2548                         id = buffer_release( _id );
2549                 }
2550
2551                 // Find quietness specification, if present
2552                 const char* quiet_str = NULL;
2553                 if( options ) {
2554                         const jsonObject* quiet_obj = jsonObjectGetKeyConst( options, "quiet" );
2555                         if( quiet_obj )
2556                                 quiet_str = jsonObjectGetString( quiet_obj );
2557                 }
2558
2559                 if( str_is_true( quiet_str )) {  // if quietness is specified
2560                         obj = jsonNewObject( id );
2561                 }
2562                 else {
2563
2564                         // Fetch the row that we just inserted, so that we can return it to the client
2565                         jsonObject* where_clause = jsonNewObjectType( JSON_HASH );
2566                         jsonObjectSetKey( where_clause, pkey, jsonNewObject( id ));
2567
2568                         int err = 0;
2569                         jsonObject* list = doFieldmapperSearch( ctx, meta, where_clause, NULL, &err );
2570                         if( err )
2571                                 rc = -1;
2572                         else
2573                                 obj = jsonObjectClone( jsonObjectGetIndex( list, 0 ));
2574
2575                         jsonObjectFree( list );
2576                         jsonObjectFree( where_clause );
2577                 }
2578
2579                 free( id );
2580         }
2581
2582         free( query );
2583         osrfAppRespondComplete( ctx, obj );
2584         jsonObjectFree( obj );
2585         return rc;
2586 }
2587
2588 /**
2589         @brief Implement the retrieve method.
2590         @param ctx Pointer to the method context.
2591         @param err Pointer through which to return an error code.
2592         @return If successful, a pointer to the result to be returned to the client;
2593         otherwise NULL.
2594
2595         From the method's class, fetch a row with a specified value in the primary key.  This
2596         method relies on the database design convention that a primary key consists of a single
2597         column.
2598
2599         Method parameters:
2600         - authkey (PCRUD only)
2601         - value of the primary key for the desired row, for building the WHERE clause
2602         - a JSON_HASH containing any other SQL clauses: select, join, etc.
2603
2604         Return to client: One row from the query.
2605 */
2606 int doRetrieve( osrfMethodContext* ctx ) {
2607         if(osrfMethodVerifyContext( ctx )) {
2608                 osrfLogError( OSRF_LOG_MARK, "Invalid method context" );
2609                 return -1;
2610         }
2611
2612         if( enforce_pcrud )
2613                 timeout_needs_resetting = 1;
2614
2615         int id_pos = 0;
2616         int order_pos = 1;
2617
2618         if( enforce_pcrud ) {
2619                 id_pos = 1;
2620                 order_pos = 2;
2621         }
2622
2623         // Get the class metadata
2624         osrfHash* class_def = osrfHashGet( (osrfHash*) ctx->method->userData, "class" );
2625
2626         // Get the value of the primary key, from a method parameter
2627         const jsonObject* id_obj = jsonObjectGetIndex( ctx->params, id_pos );
2628
2629         osrfLogDebug(
2630                 OSRF_LOG_MARK,
2631                 "%s retrieving %s object with primary key value of %s",
2632                 modulename,
2633                 osrfHashGet( class_def, "fieldmapper" ),
2634                 jsonObjectGetString( id_obj )
2635         );
2636
2637         // Build a WHERE clause based on the key value
2638         jsonObject* where_clause = jsonNewObjectType( JSON_HASH );
2639         jsonObjectSetKey(
2640                 where_clause,
2641                 osrfHashGet( class_def, "primarykey" ),  // name of key column
2642                 jsonObjectClone( id_obj )                // value of key column
2643         );
2644
2645         jsonObject* rest_of_query = jsonObjectGetIndex( ctx->params, order_pos );
2646
2647         // Do the query
2648         int err = 0;
2649         jsonObject* list = doFieldmapperSearch( ctx, class_def, where_clause, rest_of_query, &err );
2650
2651         jsonObjectFree( where_clause );
2652         if( err ) {
2653                 osrfAppRespondComplete( ctx, NULL );
2654                 return -1;
2655         }
2656
2657         jsonObject* obj = jsonObjectExtractIndex( list, 0 );
2658         jsonObjectFree( list );
2659
2660         if( enforce_pcrud ) {
2661                 // no result, skip this entirely
2662                 if(NULL != obj && !verifyObjectPCRUD( ctx, class_def, obj, 1 )) {
2663                         jsonObjectFree( obj );
2664
2665                         growing_buffer* msg = buffer_init( 128 );
2666                         OSRF_BUFFER_ADD( msg, modulename );
2667                         OSRF_BUFFER_ADD( msg, ": Insufficient permissions to retrieve object" );
2668
2669                         char* m = buffer_release( msg );
2670                         osrfAppSessionStatus( ctx->session, OSRF_STATUS_NOTALLOWED, "osrfMethodException",
2671                                         ctx->request, m );
2672                         free( m );
2673
2674                         osrfAppRespondComplete( ctx, NULL );
2675                         return -1;
2676                 }
2677         }
2678
2679         // doFieldmapperSearch() now does the responding for us
2680         //osrfAppRespondComplete( ctx, obj );
2681         osrfAppRespondComplete( ctx, NULL );
2682
2683         jsonObjectFree( obj );
2684         return 0;
2685 }
2686
2687 /**
2688         @brief Translate a numeric value to a string representation for the database.
2689         @param field Pointer to the IDL field definition.
2690         @param value Pointer to a jsonObject holding the value of a field.
2691         @return Pointer to a newly allocated string.
2692
2693         The input object is typically a JSON_NUMBER, but it may be a JSON_STRING as long as
2694         its contents are numeric.  A non-numeric string is likely to result in invalid SQL.
2695
2696         If the datatype of the receiving field is not numeric, wrap the value in quotes.
2697
2698         The calling code is responsible for freeing the resulting string by calling free().
2699 */
2700 static char* jsonNumberToDBString( osrfHash* field, const jsonObject* value ) {
2701         growing_buffer* val_buf = buffer_init( 32 );
2702
2703         // If the value is a number and the DB field is numeric, no quotes needed
2704         if( value->type == JSON_NUMBER && !strcmp( get_primitive( field ), "number") ) {
2705                 buffer_fadd( val_buf, jsonObjectGetString( value ) );
2706         } else {
2707                 // Presumably this was really intended to be a string, so quote it
2708                 char* str = jsonObjectToSimpleString( value );
2709                 if( dbi_conn_quote_string( dbhandle, &str )) {
2710                         OSRF_BUFFER_ADD( val_buf, str );
2711                         free( str );
2712                 } else {
2713                         osrfLogError( OSRF_LOG_MARK, "%s: Error quoting key string [%s]", modulename, str );
2714                         free( str );
2715                         buffer_free( val_buf );
2716                         return NULL;
2717                 }
2718         }
2719
2720         return buffer_release( val_buf );
2721 }
2722
2723 static char* searchINPredicate( const char* class_alias, osrfHash* field,
2724                 jsonObject* node, const char* op, osrfMethodContext* ctx ) {
2725         growing_buffer* sql_buf = buffer_init( 32 );
2726
2727         buffer_fadd(
2728                 sql_buf,
2729                 "\"%s\".%s ",
2730                 class_alias,
2731                 osrfHashGet( field, "name" )
2732         );
2733
2734         if( !op ) {
2735                 buffer_add( sql_buf, "IN (" );
2736         } else if( !strcasecmp( op,"not in" )) {
2737                 buffer_add( sql_buf, "NOT IN (" );
2738         } else {
2739                 buffer_add( sql_buf, "IN (" );
2740         }
2741
2742         if( node->type == JSON_HASH ) {
2743                 // subquery predicate
2744                 char* subpred = buildQuery( ctx, node, SUBSELECT );
2745                 if( ! subpred ) {
2746                         buffer_free( sql_buf );
2747                         return NULL;
2748                 }
2749
2750                 buffer_add( sql_buf, subpred );
2751                 free( subpred );
2752
2753         } else if( node->type == JSON_ARRAY ) {
2754                 // literal value list
2755                 int in_item_index = 0;
2756                 int in_item_first = 1;
2757                 const jsonObject* in_item;
2758                 while( (in_item = jsonObjectGetIndex( node, in_item_index++ )) ) {
2759
2760                         if( in_item_first )
2761                                 in_item_first = 0;
2762                         else
2763                                 buffer_add( sql_buf, ", " );
2764
2765                         // Sanity check
2766                         if( in_item->type != JSON_STRING && in_item->type != JSON_NUMBER ) {
2767                                 osrfLogError( OSRF_LOG_MARK,
2768                                                 "%s: Expected string or number within IN list; found %s",
2769                                                 modulename, json_type( in_item->type ) );
2770                                 buffer_free( sql_buf );
2771                                 return NULL;
2772                         }
2773
2774                         // Append the literal value -- quoted if not a number
2775                         if( JSON_NUMBER == in_item->type ) {
2776                                 char* val = jsonNumberToDBString( field, in_item );
2777                                 OSRF_BUFFER_ADD( sql_buf, val );
2778                                 free( val );
2779
2780                         } else if( !strcmp( get_primitive( field ), "number" )) {
2781                                 char* val = jsonNumberToDBString( field, in_item );
2782                                 OSRF_BUFFER_ADD( sql_buf, val );
2783                                 free( val );
2784
2785                         } else {
2786                                 char* key_string = jsonObjectToSimpleString( in_item );
2787                                 if( dbi_conn_quote_string( dbhandle, &key_string )) {
2788                                         OSRF_BUFFER_ADD( sql_buf, key_string );
2789                                         free( key_string );
2790                                 } else {
2791                                         osrfLogError( OSRF_LOG_MARK,
2792                                                         "%s: Error quoting key string [%s]", modulename, key_string );
2793                                         free( key_string );
2794                                         buffer_free( sql_buf );
2795                                         return NULL;
2796                                 }
2797                         }
2798                 }
2799
2800                 if( in_item_first ) {
2801                         osrfLogError(OSRF_LOG_MARK, "%s: Empty IN list", modulename );
2802                         buffer_free( sql_buf );
2803                         return NULL;
2804                 }
2805         } else {
2806                 osrfLogError( OSRF_LOG_MARK, "%s: Expected object or array for IN clause; found %s",
2807                         modulename, json_type( node->type ));
2808                 buffer_free( sql_buf );
2809                 return NULL;
2810         }
2811
2812         OSRF_BUFFER_ADD_CHAR( sql_buf, ')' );
2813
2814         return buffer_release( sql_buf );
2815 }
2816
2817 // Receive a JSON_ARRAY representing a function call.  The first
2818 // entry in the array is the function name.  The rest are parameters.
2819 static char* searchValueTransform( const jsonObject* array ) {
2820
2821         if( array->size < 1 ) {
2822                 osrfLogError( OSRF_LOG_MARK, "%s: Empty array for value transform", modulename );
2823                 return NULL;
2824         }
2825
2826         // Get the function name
2827         jsonObject* func_item = jsonObjectGetIndex( array, 0 );
2828         if( func_item->type != JSON_STRING ) {
2829                 osrfLogError( OSRF_LOG_MARK, "%s: Error: expected function name, found %s",
2830                         modulename, json_type( func_item->type ));
2831                 return NULL;
2832         }
2833
2834         growing_buffer* sql_buf = buffer_init( 32 );
2835
2836         OSRF_BUFFER_ADD( sql_buf, jsonObjectGetString( func_item ) );
2837         OSRF_BUFFER_ADD( sql_buf, "( " );
2838
2839         // Get the parameters
2840         int func_item_index = 1;   // We already grabbed the zeroth entry
2841         while( (func_item = jsonObjectGetIndex( array, func_item_index++ )) ) {
2842
2843                 // Add a separator comma, if we need one
2844                 if( func_item_index > 2 )
2845                         buffer_add( sql_buf, ", " );
2846
2847                 // Add the current parameter
2848                 if( func_item->type == JSON_NULL ) {
2849                         buffer_add( sql_buf, "NULL" );
2850                 } else {
2851                         if( func_item->type == JSON_BOOL ) {
2852                                 if( jsonBoolIsTrue(func_item) ) {
2853                                         buffer_add( sql_buf, "TRUE" );
2854                                 } else {
2855                                         buffer_add( sql_buf, "FALSE" );
2856                                 }
2857                         } else {
2858                                 char* val = jsonObjectToSimpleString( func_item );
2859                                 if( dbi_conn_quote_string( dbhandle, &val )) {
2860                                         OSRF_BUFFER_ADD( sql_buf, val );
2861                                         free( val );
2862                                 } else {
2863                                         osrfLogError( OSRF_LOG_MARK,
2864                                                 "%s: Error quoting key string [%s]", modulename, val );
2865                                         buffer_free( sql_buf );
2866                                         free( val );
2867                                         return NULL;
2868                                 }
2869                         }
2870                 }
2871         }
2872
2873         buffer_add( sql_buf, " )" );
2874
2875         return buffer_release( sql_buf );
2876 }
2877
2878 static char* searchFunctionPredicate( const char* class_alias, osrfHash* field,
2879                 const jsonObject* node, const char* op ) {
2880
2881         if( ! is_good_operator( op ) ) {
2882                 osrfLogError( OSRF_LOG_MARK, "%s: Invalid operator [%s]", modulename, op );
2883                 return NULL;
2884         }
2885
2886         char* val = searchValueTransform( node );
2887         if( !val )
2888                 return NULL;
2889
2890         const char* right_percent = "";
2891         const char* real_op       = op;
2892
2893         if( !strcasecmp( op, "startwith") ) {
2894                 real_op = "like";
2895                 right_percent = "|| '%'";
2896         }
2897
2898         growing_buffer* sql_buf = buffer_init( 32 );
2899         buffer_fadd(
2900                 sql_buf,
2901                 "\"%s\".%s %s %s%s",
2902                 class_alias,
2903                 osrfHashGet( field, "name" ),
2904                 real_op,
2905                 val,
2906                 right_percent
2907         );
2908
2909         free( val );
2910
2911         return buffer_release( sql_buf );
2912 }
2913
2914 // class_alias is a class name or other table alias
2915 // field is a field definition as stored in the IDL
2916 // node comes from the method parameter, and may represent an entry in the SELECT list
2917 static char* searchFieldTransform( const char* class_alias, osrfHash* field,
2918                 const jsonObject* node ) {
2919         growing_buffer* sql_buf = buffer_init( 32 );
2920
2921         const char* field_transform = jsonObjectGetString(
2922                 jsonObjectGetKeyConst( node, "transform" ) );
2923         const char* transform_subcolumn = jsonObjectGetString(
2924                 jsonObjectGetKeyConst( node, "result_field" ) );
2925
2926         if( transform_subcolumn ) {
2927                 if( ! is_identifier( transform_subcolumn ) ) {
2928                         osrfLogError( OSRF_LOG_MARK, "%s: Invalid subfield name: \"%s\"\n",
2929                                         modulename, transform_subcolumn );
2930                         buffer_free( sql_buf );
2931                         return NULL;
2932                 }
2933                 OSRF_BUFFER_ADD_CHAR( sql_buf, '(' );    // enclose transform in parentheses
2934         }
2935
2936         if( field_transform ) {
2937
2938                 if( ! is_identifier( field_transform ) ) {
2939                         osrfLogError( OSRF_LOG_MARK, "%s: Expected function name, found \"%s\"\n",
2940                                         modulename, field_transform );
2941                         buffer_free( sql_buf );
2942                         return NULL;
2943                 }
2944
2945                 if( obj_is_true( jsonObjectGetKeyConst( node, "distinct" ) ) ) {
2946                         buffer_fadd( sql_buf, "%s(DISTINCT \"%s\".%s",
2947                                 field_transform, class_alias, osrfHashGet( field, "name" ));
2948                 } else {
2949                         buffer_fadd( sql_buf, "%s(\"%s\".%s",
2950                                 field_transform, class_alias, osrfHashGet( field, "name" ));
2951                 }
2952
2953                 const jsonObject* array = jsonObjectGetKeyConst( node, "params" );
2954
2955                 if( array ) {
2956                         if( array->type != JSON_ARRAY ) {
2957                                 osrfLogError( OSRF_LOG_MARK,
2958                                         "%s: Expected JSON_ARRAY for function params; found %s",
2959                                         modulename, json_type( array->type ) );
2960                                 buffer_free( sql_buf );
2961                                 return NULL;
2962                         }
2963                         int func_item_index = 0;
2964                         jsonObject* func_item;
2965                         while( (func_item = jsonObjectGetIndex( array, func_item_index++ ))) {
2966
2967                                 char* val = jsonObjectToSimpleString( func_item );
2968
2969                                 if( !val ) {
2970                                         buffer_add( sql_buf, ",NULL" );
2971                                 } else if( dbi_conn_quote_string( dbhandle, &val )) {
2972                                         OSRF_BUFFER_ADD_CHAR( sql_buf, ',' );
2973                                         OSRF_BUFFER_ADD( sql_buf, val );
2974                                 } else {
2975                                         osrfLogError( OSRF_LOG_MARK,
2976                                                         "%s: Error quoting key string [%s]", modulename, val );
2977                                         free( val );
2978                                         buffer_free( sql_buf );
2979                                         return NULL;
2980                                 }
2981                                 free( val );
2982                         }
2983                 }
2984
2985                 buffer_add( sql_buf, " )" );
2986
2987         } else {
2988                 buffer_fadd( sql_buf, "\"%s\".%s", class_alias, osrfHashGet( field, "name" ));
2989         }
2990
2991         if( transform_subcolumn )
2992                 buffer_fadd( sql_buf, ").\"%s\"", transform_subcolumn );
2993
2994         return buffer_release( sql_buf );
2995 }
2996
2997 static char* searchFieldTransformPredicate( const ClassInfo* class_info, osrfHash* field,
2998                 const jsonObject* node, const char* op ) {
2999
3000         if( ! is_good_operator( op ) ) {
3001                 osrfLogError( OSRF_LOG_MARK, "%s: Error: Invalid operator %s", modulename, op );
3002                 return NULL;
3003         }
3004
3005         char* field_transform = searchFieldTransform( class_info->alias, field, node );
3006         if( ! field_transform )
3007                 return NULL;
3008         char* value = NULL;
3009         int extra_parens = 0;   // boolean
3010
3011         const jsonObject* value_obj = jsonObjectGetKeyConst( node, "value" );
3012         if( ! value_obj ) {
3013                 value = searchWHERE( node, class_info, AND_OP_JOIN, NULL );
3014                 if( !value ) {
3015                         osrfLogError( OSRF_LOG_MARK, "%s: Error building condition for field transform",
3016                                 modulename );
3017                         free( field_transform );
3018                         return NULL;
3019                 }
3020                 extra_parens = 1;
3021         } else if( value_obj->type == JSON_ARRAY ) {
3022                 value = searchValueTransform( value_obj );
3023                 if( !value ) {
3024                         osrfLogError( OSRF_LOG_MARK,
3025                                 "%s: Error building value transform for field transform", modulename );
3026                         free( field_transform );
3027                         return NULL;
3028                 }
3029         } else if( value_obj->type == JSON_HASH ) {
3030                 value = searchWHERE( value_obj, class_info, AND_OP_JOIN, NULL );
3031                 if( !value ) {
3032                         osrfLogError( OSRF_LOG_MARK, "%s: Error building predicate for field transform",
3033                                 modulename );
3034                         free( field_transform );
3035                         return NULL;
3036                 }
3037                 extra_parens = 1;
3038         } else if( value_obj->type == JSON_NUMBER ) {
3039                 value = jsonNumberToDBString( field, value_obj );
3040         } else if( value_obj->type == JSON_NULL ) {
3041                 osrfLogError( OSRF_LOG_MARK,
3042                         "%s: Error building predicate for field transform: null value", modulename );
3043                 free( field_transform );
3044                 return NULL;
3045         } else if( value_obj->type == JSON_BOOL ) {
3046                 osrfLogError( OSRF_LOG_MARK,
3047                         "%s: Error building predicate for field transform: boolean value", modulename );
3048                 free( field_transform );
3049                 return NULL;
3050         } else {
3051                 if( !strcmp( get_primitive( field ), "number") ) {
3052                         value = jsonNumberToDBString( field, value_obj );
3053                 } else {
3054                         value = jsonObjectToSimpleString( value_obj );
3055                         if( !dbi_conn_quote_string( dbhandle, &value )) {
3056                                 osrfLogError( OSRF_LOG_MARK, "%s: Error quoting key string [%s]",
3057                                         modulename, value );
3058                                 free( value );
3059                                 free( field_transform );
3060                                 return NULL;
3061                         }
3062                 }
3063         }
3064
3065         const char* left_parens  = "";
3066         const char* right_parens = "";
3067
3068         if( extra_parens ) {
3069                 left_parens  = "(";
3070                 right_parens = ")";
3071         }
3072
3073         const char* right_percent = "";
3074         const char* real_op       = op;
3075
3076         if( !strcasecmp( op, "startwith") ) {
3077                 real_op = "like";
3078                 right_percent = "|| '%'";
3079         }
3080
3081         growing_buffer* sql_buf = buffer_init( 32 );
3082
3083         buffer_fadd(
3084                 sql_buf,
3085                 "%s%s %s %s %s%s %s%s",
3086                 left_parens,
3087                 field_transform,
3088                 real_op,
3089                 left_parens,
3090                 value,
3091                 right_percent,
3092                 right_parens,
3093                 right_parens
3094         );
3095
3096         free( value );
3097         free( field_transform );
3098
3099         return buffer_release( sql_buf );
3100 }
3101
3102 static char* searchSimplePredicate( const char* op, const char* class_alias,
3103                 osrfHash* field, const jsonObject* node ) {
3104
3105         if( ! is_good_operator( op ) ) {
3106                 osrfLogError( OSRF_LOG_MARK, "%s: Invalid operator [%s]", modulename, op );
3107                 return NULL;
3108         }
3109
3110         char* val = NULL;
3111
3112         // Get the value to which we are comparing the specified column
3113         if( node->type != JSON_NULL ) {
3114                 if( node->type == JSON_NUMBER ) {
3115                         val = jsonNumberToDBString( field, node );
3116                 } else if( !strcmp( get_primitive( field ), "number" ) ) {
3117                         val = jsonNumberToDBString( field, node );
3118                 } else {
3119                         val = jsonObjectToSimpleString( node );
3120                 }
3121         }
3122
3123         if( val ) {
3124                 if( JSON_NUMBER != node->type && strcmp( get_primitive( field ), "number") ) {
3125                         // Value is not numeric; enclose it in quotes
3126                         if( !dbi_conn_quote_string( dbhandle, &val ) ) {
3127                                 osrfLogError( OSRF_LOG_MARK, "%s: Error quoting key string [%s]",
3128                                         modulename, val );
3129                                 free( val );
3130                                 return NULL;
3131                         }
3132                 }
3133         } else {
3134                 // Compare to a null value
3135                 val = strdup( "NULL" );
3136                 if( strcmp( op, "=" ))
3137                         op = "IS NOT";
3138                 else
3139                         op = "IS";
3140         }
3141
3142         const char* right_percent = "";
3143         const char* real_op       = op;
3144
3145         if( !strcasecmp( op, "startwith") ) {
3146                 real_op = "like";
3147                 right_percent = "|| '%'";
3148         }
3149
3150         growing_buffer* sql_buf = buffer_init( 32 );
3151         buffer_fadd( sql_buf, "\"%s\".%s %s %s%s", class_alias, osrfHashGet(field, "name"), real_op, val, right_percent );
3152         char* pred = buffer_release( sql_buf );
3153
3154         free( val );
3155
3156         return pred;
3157 }
3158
3159 static char* searchBETWEENPredicate( const char* class_alias,
3160                 osrfHash* field, const jsonObject* node ) {
3161
3162         const jsonObject* x_node = jsonObjectGetIndex( node, 0 );
3163         const jsonObject* y_node = jsonObjectGetIndex( node, 1 );
3164
3165         if( NULL == y_node ) {
3166                 osrfLogError( OSRF_LOG_MARK, "%s: Not enough operands for BETWEEN operator", modulename );
3167                 return NULL;
3168         }
3169         else if( NULL != jsonObjectGetIndex( node, 2 ) ) {
3170                 osrfLogError( OSRF_LOG_MARK, "%s: Too many operands for BETWEEN operator", modulename );
3171                 return NULL;
3172         }
3173
3174         char* x_string;
3175         char* y_string;
3176
3177         if( !strcmp( get_primitive( field ), "number") ) {
3178                 x_string = jsonNumberToDBString( field, x_node );
3179                 y_string = jsonNumberToDBString( field, y_node );
3180
3181         } else {
3182                 x_string = jsonObjectToSimpleString( x_node );
3183                 y_string = jsonObjectToSimpleString( y_node );
3184                 if( !(dbi_conn_quote_string( dbhandle, &x_string )
3185                         && dbi_conn_quote_string( dbhandle, &y_string )) ) {
3186                         osrfLogError( OSRF_LOG_MARK, "%s: Error quoting key strings [%s] and [%s]",
3187                                         modulename, x_string, y_string );
3188                         free( x_string );
3189                         free( y_string );
3190                         return NULL;
3191                 }
3192         }
3193
3194         growing_buffer* sql_buf = buffer_init( 32 );
3195         buffer_fadd( sql_buf, "\"%s\".%s BETWEEN %s AND %s",
3196                         class_alias, osrfHashGet( field, "name" ), x_string, y_string );
3197         free( x_string );
3198         free( y_string );
3199
3200         return buffer_release( sql_buf );
3201 }
3202
3203 static char* searchPredicate( const ClassInfo* class_info, osrfHash* field,
3204                                                           jsonObject* node, osrfMethodContext* ctx ) {
3205
3206         char* pred = NULL;
3207         if( node->type == JSON_ARRAY ) { // equality IN search
3208                 pred = searchINPredicate( class_info->alias, field, node, NULL, ctx );
3209         } else if( node->type == JSON_HASH ) { // other search
3210                 jsonIterator* pred_itr = jsonNewIterator( node );
3211                 if( !jsonIteratorHasNext( pred_itr ) ) {
3212                         osrfLogError( OSRF_LOG_MARK, "%s: Empty predicate for field \"%s\"",
3213                                         modulename, osrfHashGet(field, "name" ));
3214                 } else {
3215                         jsonObject* pred_node = jsonIteratorNext( pred_itr );
3216
3217                         // Verify that there are no additional predicates
3218                         if( jsonIteratorHasNext( pred_itr ) ) {
3219                                 osrfLogError( OSRF_LOG_MARK, "%s: Multiple predicates for field \"%s\"",
3220                                                 modulename, osrfHashGet(field, "name" ));
3221                         } else if( !(strcasecmp( pred_itr->key,"between" )) )
3222                                 pred = searchBETWEENPredicate( class_info->alias, field, pred_node );
3223                         else if( !(strcasecmp( pred_itr->key,"in" ))
3224                                         || !(strcasecmp( pred_itr->key,"not in" )) )
3225                                 pred = searchINPredicate(
3226                                         class_info->alias, field, pred_node, pred_itr->key, ctx );
3227                         else if( pred_node->type == JSON_ARRAY )
3228                                 pred = searchFunctionPredicate(
3229                                         class_info->alias, field, pred_node, pred_itr->key );
3230                         else if( pred_node->type == JSON_HASH )
3231                                 pred = searchFieldTransformPredicate(
3232                                         class_info, field, pred_node, pred_itr->key );
3233                         else
3234                                 pred = searchSimplePredicate( pred_itr->key, class_info->alias, field, pred_node );
3235                 }
3236                 jsonIteratorFree( pred_itr );
3237
3238         } else if( node->type == JSON_NULL ) { // IS NULL search
3239                 growing_buffer* _p = buffer_init( 64 );
3240                 buffer_fadd(
3241                         _p,
3242                         "\"%s\".%s IS NULL",
3243                         class_info->alias,
3244                         osrfHashGet( field, "name" )
3245                 );
3246                 pred = buffer_release( _p );
3247         } else { // equality search
3248                 pred = searchSimplePredicate( "=", class_info->alias, field, node );
3249         }
3250
3251         return pred;
3252
3253 }
3254
3255
3256 /*
3257
3258 join : {
3259         acn : {
3260                 field : record,
3261                 fkey : id
3262                 type : left
3263                 filter_op : or
3264                 filter : { ... },
3265                 join : {
3266                         acp : {
3267                                 field : call_number,
3268                                 fkey : id,
3269                                 filter : { ... },
3270                         },
3271                 },
3272         },
3273         mrd : {
3274                 field : record,
3275                 type : inner
3276                 fkey : id,
3277                 filter : { ... },
3278         }
3279 }
3280
3281   Or, to specify join order:
3282
3283 join : [
3284     {mrd:{field:'record', type:'inner'}},
3285     {acn:{field:'record', type:'left'}}
3286 ]
3287
3288 */
3289
3290 static char* searchJOIN( const jsonObject* join_hash, const ClassInfo* left_info ) {
3291
3292         jsonObject* working_hash;
3293         jsonObject* freeable_hash = NULL;
3294
3295         jsonObject* working_array;
3296         jsonObject* freeable_array = NULL;
3297
3298         if( join_hash->type == JSON_ARRAY ) {
3299                 working_array = (jsonObject*)join_hash;
3300         } else {
3301                 working_array = jsonNewObjectType( JSON_ARRAY );
3302
3303                 if( join_hash->type == JSON_HASH ) {
3304                         working_hash = (jsonObject*)join_hash;
3305                 } else if( join_hash->type == JSON_STRING ) {
3306                     freeable_array = working_array;
3307                         // turn it into a JSON_HASH by creating a wrapper
3308                         // around a copy of the original
3309                         const char* _tmp = jsonObjectGetString( join_hash );
3310                         freeable_hash = jsonNewObjectType( JSON_HASH );
3311                         jsonObjectSetKey( freeable_hash, _tmp, NULL );
3312                         working_hash = freeable_hash;
3313                 } else {
3314                         osrfLogError(
3315                                 OSRF_LOG_MARK,
3316                                 "%s: JOIN failed; expected JSON object type not found",
3317                                 modulename
3318                         );
3319                         return NULL;
3320                 }
3321
3322                 jsonObjectPush( working_array, working_hash );
3323         }
3324
3325         growing_buffer* join_buf = buffer_init( 128 );
3326         const char* leftclass = left_info->class_name;
3327
3328         unsigned long order_idx = 0;
3329         while(( working_hash = jsonObjectGetIndex( working_array, order_idx++ ) )) {
3330
3331             jsonObject* freeable_subhash = NULL;
3332                 if( working_hash->type == JSON_STRING ) {
3333                         // turn it into a JSON_HASH by creating a wrapper
3334                         // around a copy of the original
3335                         const char* _inner_tmp = jsonObjectGetString( working_hash );
3336                         freeable_subhash = jsonNewObjectType( JSON_HASH );
3337                         jsonObjectSetKey( freeable_subhash, _inner_tmp, NULL );
3338                         working_hash = freeable_subhash;
3339                 }
3340
3341                 jsonObject* snode = NULL;
3342                 jsonIterator* search_itr = jsonNewIterator( working_hash );
3343
3344                 while ( (snode = jsonIteratorNext( search_itr )) ) {
3345                         const char* right_alias = search_itr->key;
3346                         const char* class =
3347                                         jsonObjectGetString( jsonObjectGetKeyConst( snode, "class" ) );
3348                         if( ! class )
3349                                 class = right_alias;
3350
3351                         const ClassInfo* right_info = add_joined_class( right_alias, class );
3352                         if( !right_info ) {
3353                                 osrfLogError(
3354                                         OSRF_LOG_MARK,
3355                                         "%s: JOIN failed.  Class \"%s\" not resolved in IDL",
3356                                         modulename,
3357                                         search_itr->key
3358                                 );
3359                                 jsonIteratorFree( search_itr );
3360                                 buffer_free( join_buf );
3361                                 if( freeable_subhash )
3362                                         jsonObjectFree( freeable_subhash );
3363                                 if( freeable_hash )
3364                                         jsonObjectFree( freeable_hash );
3365                                 if( freeable_array )
3366                                         jsonObjectFree( freeable_array );
3367                                 return NULL;
3368                         }
3369                         osrfHash* links = right_info->links;
3370                         const char* table  = right_info->source_def;
3371
3372                         const char* fkey  = jsonObjectGetString( jsonObjectGetKeyConst( snode, "fkey" ) );
3373                         const char* field = jsonObjectGetString( jsonObjectGetKeyConst( snode, "field" ) );
3374
3375                         if( field && !fkey ) {
3376                                 // Look up the corresponding join column in the IDL.
3377                                 // The link must be defined in the child table,
3378                                 // and point to the right parent table.
3379                                 osrfHash* idl_link = (osrfHash*) osrfHashGet( links, field );
3380                                 const char* reltype = NULL;
3381                                 const char* other_class = NULL;
3382                                 reltype = osrfHashGet( idl_link, "reltype" );
3383                                 if( reltype && strcmp( reltype, "has_many" ) )
3384                                         other_class = osrfHashGet( idl_link, "class" );
3385                                 if( other_class && !strcmp( other_class, leftclass ) )
3386                                         fkey = osrfHashGet( idl_link, "key" );
3387                                 if( !fkey ) {
3388                                         osrfLogError(
3389                                                 OSRF_LOG_MARK,
3390                                                 "%s: JOIN failed.  No link defined from %s.%s to %s",
3391                                                 modulename,
3392                                                 class,
3393                                                 field,
3394                                                 leftclass
3395                                         );
3396                                         buffer_free( join_buf );
3397                                         if( freeable_subhash )
3398                                                 jsonObjectFree( freeable_subhash );
3399                                         if( freeable_hash )
3400                                                 jsonObjectFree( freeable_hash );
3401                                         if( freeable_array )
3402                                                 jsonObjectFree( freeable_array );
3403                                         jsonIteratorFree( search_itr );
3404                                         return NULL;
3405                                 }
3406
3407                         } else if( !field && fkey ) {
3408                                 // Look up the corresponding join column in the IDL.
3409                                 // The link must be defined in the child table,
3410                                 // and point to the right parent table.
3411                                 osrfHash* left_links = left_info->links;
3412                                 osrfHash* idl_link = (osrfHash*) osrfHashGet( left_links, fkey );
3413                                 const char* reltype = NULL;
3414                                 const char* other_class = NULL;
3415                                 reltype = osrfHashGet( idl_link, "reltype" );
3416                                 if( reltype && strcmp( reltype, "has_many" ) )
3417                                         other_class = osrfHashGet( idl_link, "class" );
3418                                 if( other_class && !strcmp( other_class, class ) )
3419                                         field = osrfHashGet( idl_link, "key" );
3420                                 if( !field ) {
3421                                         osrfLogError(
3422                                                 OSRF_LOG_MARK,
3423                                                 "%s: JOIN failed.  No link defined from %s.%s to %s",
3424                                                 modulename,
3425                                                 leftclass,
3426                                                 fkey,
3427                                                 class
3428                                         );
3429                                         buffer_free( join_buf );
3430                                         if( freeable_subhash )
3431                                                 jsonObjectFree( freeable_subhash );
3432                                         if( freeable_hash )
3433                                                 jsonObjectFree( freeable_hash );
3434                                         if( freeable_array )
3435                                                 jsonObjectFree( freeable_array );
3436                                         jsonIteratorFree( search_itr );
3437                                         return NULL;
3438                                 }
3439
3440                         } else if( !field && !fkey ) {
3441                                 osrfHash* left_links = left_info->links;
3442
3443                                 // For each link defined for the left class:
3444                                 // see if the link references the joined class
3445                                 osrfHashIterator* itr = osrfNewHashIterator( left_links );
3446                                 osrfHash* curr_link = NULL;
3447                                 while( (curr_link = osrfHashIteratorNext( itr ) ) ) {
3448                                         const char* other_class = osrfHashGet( curr_link, "class" );
3449                                         if( other_class && !strcmp( other_class, class ) ) {
3450
3451                                                 // In the IDL, the parent class doesn't always know then names of the child
3452                                                 // columns that are pointing to it, so don't use that end of the link
3453                                                 const char* reltype = osrfHashGet( curr_link, "reltype" );
3454                                                 if( reltype && strcmp( reltype, "has_many" ) ) {
3455                                                         // Found a link between the classes
3456                                                         fkey = osrfHashIteratorKey( itr );
3457                                                         field = osrfHashGet( curr_link, "key" );
3458                                                         break;
3459                                                 }
3460                                         }
3461                                 }
3462                                 osrfHashIteratorFree( itr );
3463
3464                                 if( !field || !fkey ) {
3465                                         // Do another such search, with the classes reversed
3466
3467                                         // For each link defined for the joined class:
3468                                         // see if the link references the left class
3469                                         osrfHashIterator* itr = osrfNewHashIterator( links );
3470                                         osrfHash* curr_link = NULL;
3471                                         while( (curr_link = osrfHashIteratorNext( itr ) ) ) {
3472                                                 const char* other_class = osrfHashGet( curr_link, "class" );
3473                                                 if( other_class && !strcmp( other_class, leftclass ) ) {
3474
3475                                                         // In the IDL, the parent class doesn't know then names of the child
3476                                                         // columns that are pointing to it, so don't use that end of the link
3477                                                         const char* reltype = osrfHashGet( curr_link, "reltype" );
3478                                                         if( reltype && strcmp( reltype, "has_many" ) ) {
3479                                                                 // Found a link between the classes
3480                                                                 field = osrfHashIteratorKey( itr );
3481                                                                 fkey = osrfHashGet( curr_link, "key" );
3482                                                                 break;
3483                                                         }
3484                                                 }
3485                                         }
3486                                         osrfHashIteratorFree( itr );
3487                                 }
3488
3489                                 if( !field || !fkey ) {
3490                                         osrfLogError(
3491                                                 OSRF_LOG_MARK,
3492                                                 "%s: JOIN failed.  No link defined between %s and %s",
3493                                                 modulename,
3494                                                 leftclass,
3495                                                 class
3496                                         );
3497                                         buffer_free( join_buf );
3498                                         if( freeable_subhash )
3499                                                 jsonObjectFree( freeable_subhash );
3500                                         if( freeable_hash )
3501                                                 jsonObjectFree( freeable_hash );
3502                                         if( freeable_array )
3503                                                 jsonObjectFree( freeable_array );
3504                                         jsonIteratorFree( search_itr );
3505                                         return NULL;
3506                                 }
3507                         }
3508
3509                         const char* type = jsonObjectGetString( jsonObjectGetKeyConst( snode, "type" ) );
3510                         if( type ) {
3511                                 if( !strcasecmp( type,"left" )) {
3512                                         buffer_add( join_buf, " LEFT JOIN" );
3513                                 } else if( !strcasecmp( type,"right" )) {
3514                                         buffer_add( join_buf, " RIGHT JOIN" );
3515                                 } else if( !strcasecmp( type,"full" )) {
3516                                         buffer_add( join_buf, " FULL JOIN" );
3517                                 } else {
3518                                         buffer_add( join_buf, " INNER JOIN" );
3519                                 }
3520                         } else {
3521                                 buffer_add( join_buf, " INNER JOIN" );
3522                         }
3523
3524                         buffer_fadd( join_buf, " %s AS \"%s\" ON ( \"%s\".%s = \"%s\".%s",
3525                                                 table, right_alias, right_alias, field, left_info->alias, fkey );
3526
3527                         // Add any other join conditions as specified by "filter"
3528                         const jsonObject* filter = jsonObjectGetKeyConst( snode, "filter" );
3529                         if( filter ) {
3530                                 const char* filter_op = jsonObjectGetString(
3531                                         jsonObjectGetKeyConst( snode, "filter_op" ) );
3532                                 if( filter_op && !strcasecmp( "or",filter_op )) {
3533                                         buffer_add( join_buf, " OR " );
3534                                 } else {
3535                                         buffer_add( join_buf, " AND " );
3536                                 }
3537
3538                                 char* jpred = searchWHERE( filter, right_info, AND_OP_JOIN, NULL );
3539                                 if( jpred ) {
3540                                         OSRF_BUFFER_ADD_CHAR( join_buf, ' ' );
3541                                         OSRF_BUFFER_ADD( join_buf, jpred );
3542                                         free( jpred );
3543                                 } else {
3544                                         osrfLogError(
3545                                                 OSRF_LOG_MARK,
3546                                                 "%s: JOIN failed.  Invalid conditional expression.",
3547                                                 modulename
3548                                         );
3549                                         jsonIteratorFree( search_itr );
3550                                         buffer_free( join_buf );
3551                                         if( freeable_subhash )
3552                                                 jsonObjectFree( freeable_subhash );
3553                                         if( freeable_hash )
3554                                                 jsonObjectFree( freeable_hash );
3555                                         if( freeable_array )
3556                                                 jsonObjectFree( freeable_array );
3557                                         return NULL;
3558                                 }
3559                         }
3560
3561                         buffer_add( join_buf, " ) " );
3562
3563                         // Recursively add a nested join, if one is present
3564                         const jsonObject* join_filter = jsonObjectGetKeyConst( snode, "join" );
3565                         if( join_filter ) {
3566                                 char* jpred = searchJOIN( join_filter, right_info );
3567                                 if( jpred ) {
3568                                         OSRF_BUFFER_ADD_CHAR( join_buf, ' ' );
3569                                         OSRF_BUFFER_ADD( join_buf, jpred );
3570                                         free( jpred );
3571                                 } else {
3572                                         osrfLogError( OSRF_LOG_MARK, "%s: Invalid nested join.", modulename );
3573                                         jsonIteratorFree( search_itr );
3574                                         buffer_free( join_buf );
3575                                         if( freeable_subhash )
3576                                                 jsonObjectFree( freeable_subhash );
3577                                         if( freeable_hash )
3578                                                 jsonObjectFree( freeable_hash );
3579                                         if( freeable_array )
3580                                                 jsonObjectFree( freeable_array );
3581                                         return NULL;
3582                                 }
3583                         }
3584                 }
3585
3586                 if( freeable_subhash )
3587                         jsonObjectFree( freeable_subhash );
3588
3589                 jsonIteratorFree( search_itr );
3590         }
3591
3592         if( freeable_hash )
3593                 jsonObjectFree( freeable_hash );
3594
3595         if( freeable_array )
3596                 jsonObjectFree( freeable_array );
3597
3598
3599         return buffer_release( join_buf );
3600 }
3601
3602 /*
3603
3604 { +class : { -or|-and : { field : { op : value }, ... } ... }, ... }
3605 { +class : { -or|-and : [ { field : { op : value }, ... }, ...] ... }, ... }
3606 [ { +class : { -or|-and : [ { field : { op : value }, ... }, ...] ... }, ... }, ... ]
3607
3608 Generate code to express a set of conditions, as for a WHERE clause.  Parameters:
3609
3610 search_hash is the JSON expression of the conditions.
3611 meta is the class definition from the IDL, for the relevant table.
3612 opjoin_type indicates whether multiple conditions, if present, should be
3613         connected by AND or OR.
3614 osrfMethodContext is loaded with all sorts of stuff, but all we do with it here is
3615         to pass it to other functions -- and all they do with it is to use the session
3616         and request members to send error messages back to the client.
3617
3618 */
3619
3620 static char* searchWHERE( const jsonObject* search_hash, const ClassInfo* class_info,
3621                 int opjoin_type, osrfMethodContext* ctx ) {
3622
3623         osrfLogDebug(
3624                 OSRF_LOG_MARK,
3625                 "%s: Entering searchWHERE; search_hash addr = %p, meta addr = %p, "
3626                 "opjoin_type = %d, ctx addr = %p",
3627                 modulename,
3628                 search_hash,
3629                 class_info->class_def,
3630                 opjoin_type,
3631                 ctx
3632         );
3633
3634         growing_buffer* sql_buf = buffer_init( 128 );
3635
3636         jsonObject* node = NULL;
3637
3638         int first = 1;
3639         if( search_hash->type == JSON_ARRAY ) {
3640                 if( 0 == search_hash->size ) {
3641                         osrfLogError(
3642                                 OSRF_LOG_MARK,
3643                                 "%s: Invalid predicate structure: empty JSON array",
3644                                 modulename
3645                         );
3646                         buffer_free( sql_buf );
3647                         return NULL;
3648                 }
3649
3650                 unsigned long i = 0;
3651                 while(( node = jsonObjectGetIndex( search_hash, i++ ) )) {
3652                         if( first ) {
3653                                 first = 0;
3654                         } else {
3655                                 if( opjoin_type == OR_OP_JOIN )
3656                                         buffer_add( sql_buf, " OR " );
3657                                 else
3658                                         buffer_add( sql_buf, " AND " );
3659                         }
3660
3661                         char* subpred = searchWHERE( node, class_info, opjoin_type, ctx );
3662                         if( ! subpred ) {
3663                                 buffer_free( sql_buf );
3664                                 return NULL;
3665                         }
3666
3667                         buffer_fadd( sql_buf, "( %s )", subpred );
3668                         free( subpred );
3669                 }
3670
3671         } else if( search_hash->type == JSON_HASH ) {
3672                 osrfLogDebug( OSRF_LOG_MARK,
3673                         "%s: In WHERE clause, condition type is JSON_HASH", modulename );
3674                 jsonIterator* search_itr = jsonNewIterator( search_hash );
3675                 if( !jsonIteratorHasNext( search_itr ) ) {
3676                         osrfLogError(
3677                                 OSRF_LOG_MARK,
3678                                 "%s: Invalid predicate structure: empty JSON object",
3679                                 modulename
3680                         );
3681                         jsonIteratorFree( search_itr );
3682                         buffer_free( sql_buf );
3683                         return NULL;
3684                 }
3685
3686                 while( (node = jsonIteratorNext( search_itr )) ) {
3687
3688                         if( first ) {
3689                                 first = 0;
3690                         } else {
3691                                 if( opjoin_type == OR_OP_JOIN )
3692                                         buffer_add( sql_buf, " OR " );
3693                                 else
3694                                         buffer_add( sql_buf, " AND " );
3695                         }
3696
3697                         if( '+' == search_itr->key[ 0 ] ) {
3698
3699                                 // This plus sign prefixes a class name or other table alias;
3700                                 // make sure the table alias is in scope
3701                                 ClassInfo* alias_info = search_all_alias( search_itr->key + 1 );
3702                                 if( ! alias_info ) {
3703                                         osrfLogError(
3704                                                          OSRF_LOG_MARK,
3705                                                         "%s: Invalid table alias \"%s\" in WHERE clause",
3706                                                         modulename,
3707                                                         search_itr->key + 1
3708                                         );
3709                                         jsonIteratorFree( search_itr );
3710                                         buffer_free( sql_buf );
3711                                         return NULL;
3712                                 }
3713
3714                                 if( node->type == JSON_STRING ) {
3715                                         // It's the name of a column; make sure it belongs to the class
3716                                         const char* fieldname = jsonObjectGetString( node );
3717                                         if( ! osrfHashGet( alias_info->fields, fieldname ) ) {
3718                                                 osrfLogError(
3719                                                         OSRF_LOG_MARK,
3720                                                         "%s: Invalid column name \"%s\" in WHERE clause "
3721                                                         "for table alias \"%s\"",
3722                                                         modulename,
3723                                                         fieldname,
3724                                                         alias_info->alias
3725                                                 );
3726                                                 jsonIteratorFree( search_itr );
3727                                                 buffer_free( sql_buf );
3728                                                 return NULL;
3729                                         }
3730
3731                                         buffer_fadd( sql_buf, " \"%s\".%s ", alias_info->alias, fieldname );
3732                                 } else {
3733                                         // It's something more complicated
3734                                         char* subpred = searchWHERE( node, alias_info, AND_OP_JOIN, ctx );
3735                                         if( ! subpred ) {
3736                                                 jsonIteratorFree( search_itr );
3737                                                 buffer_free( sql_buf );
3738                                                 return NULL;
3739                                         }
3740
3741                                         buffer_fadd( sql_buf, "( %s )", subpred );
3742                                         free( subpred );
3743                                 }
3744                         } else if( '-' == search_itr->key[ 0 ] ) {
3745                                 if( !strcasecmp( "-or", search_itr->key )) {
3746                                         char* subpred = searchWHERE( node, class_info, OR_OP_JOIN, ctx );
3747                                         if( ! subpred ) {
3748                                                 jsonIteratorFree( search_itr );
3749                                                 buffer_free( sql_buf );
3750                                                 return NULL;
3751                                         }
3752
3753                                         buffer_fadd( sql_buf, "( %s )", subpred );
3754                                         free( subpred );
3755                                 } else if( !strcasecmp( "-and", search_itr->key )) {
3756                                         char* subpred = searchWHERE( node, class_info, AND_OP_JOIN, ctx );
3757                                         if( ! subpred ) {
3758                                                 jsonIteratorFree( search_itr );
3759                                                 buffer_free( sql_buf );
3760                                                 return NULL;
3761                                         }
3762
3763                                         buffer_fadd( sql_buf, "( %s )", subpred );
3764                                         free( subpred );
3765                                 } else if( !strcasecmp("-not",search_itr->key) ) {
3766                                         char* subpred = searchWHERE( node, class_info, AND_OP_JOIN, ctx );
3767                                         if( ! subpred ) {
3768                                                 jsonIteratorFree( search_itr );
3769                                                 buffer_free( sql_buf );
3770                                                 return NULL;
3771                                         }
3772
3773                                         buffer_fadd( sql_buf, " NOT ( %s )", subpred );
3774                                         free( subpred );
3775                                 } else if( !strcasecmp( "-exists", search_itr->key )) {
3776                                         char* subpred = buildQuery( ctx, node, SUBSELECT );
3777                                         if( ! subpred ) {
3778                                                 jsonIteratorFree( search_itr );
3779                                                 buffer_free( sql_buf );
3780                                                 return NULL;
3781                                         }
3782
3783                                         buffer_fadd( sql_buf, "EXISTS ( %s )", subpred );
3784                                         free( subpred );
3785                                 } else if( !strcasecmp("-not-exists", search_itr->key )) {
3786                                         char* subpred = buildQuery( ctx, node, SUBSELECT );
3787                                         if( ! subpred ) {
3788                                                 jsonIteratorFree( search_itr );
3789                                                 buffer_free( sql_buf );
3790                                                 return NULL;
3791                                         }
3792
3793                                         buffer_fadd( sql_buf, "NOT EXISTS ( %s )", subpred );
3794                                         free( subpred );
3795                                 } else {     // Invalid "minus" operator
3796                                         osrfLogError(
3797                                                          OSRF_LOG_MARK,
3798                                                         "%s: Invalid operator \"%s\" in WHERE clause",
3799                                                         modulename,
3800                                                         search_itr->key
3801                                         );
3802                                         jsonIteratorFree( search_itr );
3803                                         buffer_free( sql_buf );
3804                                         return NULL;
3805                                 }
3806
3807                         } else {
3808
3809                                 const char* class = class_info->class_name;
3810                                 osrfHash* fields = class_info->fields;
3811                                 osrfHash* field = osrfHashGet( fields, search_itr->key );
3812
3813                                 if( !field ) {
3814                                         const char* table = class_info->source_def;
3815                                         osrfLogError(
3816                                                 OSRF_LOG_MARK,
3817                                                 "%s: Attempt to reference non-existent column \"%s\" on %s (%s)",
3818                                                 modulename,
3819                                                 search_itr->key,
3820                                                 table ? table : "?",
3821                                                 class ? class : "?"
3822                                         );
3823                                         jsonIteratorFree( search_itr );
3824                                         buffer_free( sql_buf );
3825                                         return NULL;
3826                                 }
3827
3828                                 char* subpred = searchPredicate( class_info, field, node, ctx );
3829                                 if( ! subpred ) {
3830                                         buffer_free( sql_buf );
3831                                         jsonIteratorFree( search_itr );
3832                                         return NULL;
3833                                 }
3834
3835                                 buffer_add( sql_buf, subpred );
3836                                 free( subpred );
3837                         }
3838                 }
3839                 jsonIteratorFree( search_itr );
3840
3841         } else {
3842                 // ERROR ... only hash and array allowed at this level
3843                 char* predicate_string = jsonObjectToJSON( search_hash );
3844                 osrfLogError(
3845                         OSRF_LOG_MARK,
3846                         "%s: Invalid predicate structure: %s",
3847                         modulename,
3848                         predicate_string
3849                 );
3850                 buffer_free( sql_buf );
3851                 free( predicate_string );
3852                 return NULL;
3853         }
3854
3855         return buffer_release( sql_buf );
3856 }
3857
3858 /* Build a JSON_ARRAY of field names for a given table alias
3859 */
3860 static jsonObject* defaultSelectList( const char* table_alias ) {
3861
3862         if( ! table_alias )
3863                 table_alias = "";
3864
3865         ClassInfo* class_info = search_all_alias( table_alias );
3866         if( ! class_info ) {
3867                 osrfLogError(
3868                         OSRF_LOG_MARK,
3869                         "%s: Can't build default SELECT clause for \"%s\"; no such table alias",
3870                         modulename,
3871                         table_alias
3872                 );
3873                 return NULL;
3874         }
3875
3876         jsonObject* array = jsonNewObjectType( JSON_ARRAY );
3877         osrfHash* field_def = NULL;
3878         osrfHashIterator* field_itr = osrfNewHashIterator( class_info->fields );
3879         while( ( field_def = osrfHashIteratorNext( field_itr ) ) ) {
3880                 const char* field_name = osrfHashIteratorKey( field_itr );
3881                 if( ! str_is_true( osrfHashGet( field_def, "virtual" ) ) ) {
3882                         jsonObjectPush( array, jsonNewObject( field_name ) );
3883                 }
3884         }
3885         osrfHashIteratorFree( field_itr );
3886
3887         return array;
3888 }
3889
3890 // Translate a jsonObject into a UNION, INTERSECT, or EXCEPT query.
3891 // The jsonObject must be a JSON_HASH with an single entry for "union",
3892 // "intersect", or "except".  The data associated with this key must be an
3893 // array of hashes, each hash being a query.
3894 // Also allowed but currently ignored: entries for "order_by" and "alias".
3895 static char* doCombo( osrfMethodContext* ctx, jsonObject* combo, int flags ) {
3896         // Sanity check
3897         if( ! combo || combo->type != JSON_HASH )
3898                 return NULL;      // should be impossible; validated by caller
3899
3900         const jsonObject* query_array = NULL;   // array of subordinate queries
3901         const char* op = NULL;     // name of operator, e.g. UNION
3902         const char* alias = NULL;  // alias for the query (needed for ORDER BY)
3903         int op_count = 0;          // for detecting conflicting operators
3904         int excepting = 0;         // boolean
3905         int all = 0;               // boolean
3906         jsonObject* order_obj = NULL;
3907
3908         // Identify the elements in the hash
3909         jsonIterator* query_itr = jsonNewIterator( combo );
3910         jsonObject* curr_obj = NULL;
3911         while( (curr_obj = jsonIteratorNext( query_itr ) ) ) {
3912                 if( ! strcmp( "union", query_itr->key ) ) {
3913                         ++op_count;
3914                         op = " UNION ";
3915                         query_array = curr_obj;
3916                 } else if( ! strcmp( "intersect", query_itr->key ) ) {
3917                         ++op_count;
3918                         op = " INTERSECT ";
3919                         query_array = curr_obj;
3920                 } else if( ! strcmp( "except", query_itr->key ) ) {
3921                         ++op_count;
3922                         op = " EXCEPT ";
3923                         excepting = 1;
3924                         query_array = curr_obj;
3925                 } else if( ! strcmp( "order_by", query_itr->key ) ) {
3926                         osrfLogWarning(
3927                                 OSRF_LOG_MARK,
3928                                 "%s: ORDER BY not supported for UNION, INTERSECT, or EXCEPT",
3929                                 modulename
3930                         );
3931                         order_obj = curr_obj;
3932                 } else if( ! strcmp( "alias", query_itr->key ) ) {
3933                         if( curr_obj->type != JSON_STRING ) {
3934                                 jsonIteratorFree( query_itr );
3935                                 return NULL;
3936                         }
3937                         alias = jsonObjectGetString( curr_obj );
3938                 } else if( ! strcmp( "all", query_itr->key ) ) {
3939                         if( obj_is_true( curr_obj ) )
3940                                 all = 1;
3941                 } else {
3942                         if( ctx )
3943                                 osrfAppSessionStatus(
3944                                         ctx->session,
3945                                         OSRF_STATUS_INTERNALSERVERERROR,
3946                                         "osrfMethodException",
3947                                         ctx->request,
3948                                         "Malformed query; unexpected entry in query object"
3949                                 );
3950                         osrfLogError(
3951                                 OSRF_LOG_MARK,
3952                                 "%s: Unexpected entry for \"%s\" in%squery",
3953                                 modulename,
3954                                 query_itr->key,
3955                                 op
3956                         );
3957                         jsonIteratorFree( query_itr );
3958                         return NULL;
3959                 }
3960         }
3961         jsonIteratorFree( query_itr );
3962
3963         // More sanity checks
3964         if( ! query_array ) {
3965                 if( ctx )
3966                         osrfAppSessionStatus(
3967                                 ctx->session,
3968                                 OSRF_STATUS_INTERNALSERVERERROR,
3969                                 "osrfMethodException",
3970                                 ctx->request,
3971                                 "Expected UNION, INTERSECT, or EXCEPT operator not found"
3972                         );
3973                 osrfLogError(
3974                         OSRF_LOG_MARK,
3975                         "%s: Expected UNION, INTERSECT, or EXCEPT operator not found",
3976                         modulename
3977                 );
3978                 return NULL;        // should be impossible...
3979         } else if( op_count > 1 ) {
3980                 if( ctx )
3981                                 osrfAppSessionStatus(
3982                                 ctx->session,
3983                                 OSRF_STATUS_INTERNALSERVERERROR,
3984                                 "osrfMethodException",
3985                                 ctx->request,
3986                                 "Found more than one of UNION, INTERSECT, and EXCEPT in same query"
3987                         );
3988                 osrfLogError(
3989                         OSRF_LOG_MARK,
3990                         "%s: Found more than one of UNION, INTERSECT, and EXCEPT in same query",
3991                         modulename
3992                 );
3993                 return NULL;
3994         } if( query_array->type != JSON_ARRAY ) {
3995                 if( ctx )
3996                                 osrfAppSessionStatus(
3997                                 ctx->session,
3998                                 OSRF_STATUS_INTERNALSERVERERROR,
3999                                 "osrfMethodException",
4000                                 ctx->request,
4001                                 "Malformed query: expected array of queries under UNION, INTERSECT or EXCEPT"
4002                         );
4003                 osrfLogError(
4004                         OSRF_LOG_MARK,
4005                         "%s: Expected JSON_ARRAY of queries for%soperator; found %s",
4006                         modulename,
4007                         op,
4008                         json_type( query_array->type )
4009                 );
4010                 return NULL;
4011         } if( query_array->size < 2 ) {
4012                 if( ctx )
4013                         osrfAppSessionStatus(
4014                                 ctx->session,
4015                                 OSRF_STATUS_INTERNALSERVERERROR,
4016                                 "osrfMethodException",
4017                                 ctx->request,
4018                                 "UNION, INTERSECT or EXCEPT requires multiple queries as operands"
4019                         );
4020                 osrfLogError(
4021                         OSRF_LOG_MARK,
4022                         "%s:%srequires multiple queries as operands",
4023                         modulename,
4024                         op
4025                 );
4026                 return NULL;
4027         } else if( excepting && query_array->size > 2 ) {
4028                 if( ctx )
4029                         osrfAppSessionStatus(
4030                                 ctx->session,
4031                                 OSRF_STATUS_INTERNALSERVERERROR,
4032                                 "osrfMethodException",
4033                                 ctx->request,
4034                                 "EXCEPT operator has too many queries as operands"
4035                         );
4036                 osrfLogError(
4037                         OSRF_LOG_MARK,
4038                         "%s:EXCEPT operator has too many queries as operands",
4039                         modulename
4040                 );
4041                 return NULL;
4042         } else if( order_obj && ! alias ) {
4043                 if( ctx )
4044                         osrfAppSessionStatus(
4045                                 ctx->session,
4046                                 OSRF_STATUS_INTERNALSERVERERROR,
4047                                 "osrfMethodException",
4048                                 ctx->request,
4049                                 "ORDER BY requires an alias for a UNION, INTERSECT, or EXCEPT"
4050                         );
4051                 osrfLogError(
4052                         OSRF_LOG_MARK,
4053                         "%s:ORDER BY requires an alias for a UNION, INTERSECT, or EXCEPT",
4054                         modulename
4055                 );
4056                 return NULL;
4057         }
4058
4059         // So far so good.  Now build the SQL.
4060         growing_buffer* sql = buffer_init( 256 );
4061
4062         // If we nested inside another UNION, INTERSECT, or EXCEPT,
4063         // Add a layer of parentheses
4064         if( flags & SUBCOMBO )
4065                 OSRF_BUFFER_ADD( sql, "( " );
4066
4067         // Traverse the query array.  Each entry should be a hash.
4068         int first = 1;   // boolean
4069         int i = 0;
4070         jsonObject* query = NULL;
4071         while( (query = jsonObjectGetIndex( query_array, i++ )) ) {
4072                 if( query->type != JSON_HASH ) {
4073                         if( ctx )
4074                                 osrfAppSessionStatus(
4075                                         ctx->session,
4076                                         OSRF_STATUS_INTERNALSERVERERROR,
4077                                         "osrfMethodException",
4078                                         ctx->request,
4079                                         "Malformed query under UNION, INTERSECT or EXCEPT"
4080                                 );
4081                         osrfLogError(
4082                                 OSRF_LOG_MARK,
4083                                 "%s: Malformed query under%s -- expected JSON_HASH, found %s",
4084                                 modulename,
4085                                 op,
4086                                 json_type( query->type )
4087                         );
4088                         buffer_free( sql );
4089                         return NULL;
4090                 }
4091
4092                 if( first )
4093                         first = 0;
4094                 else {
4095                         OSRF_BUFFER_ADD( sql, op );
4096                         if( all )
4097                                 OSRF_BUFFER_ADD( sql, "ALL " );
4098                 }
4099
4100                 char* query_str = buildQuery( ctx, query, SUBSELECT | SUBCOMBO );
4101                 if( ! query_str ) {
4102                         osrfLogError(
4103                                 OSRF_LOG_MARK,
4104                                 "%s: Error building query under%s",
4105                                 modulename,
4106                                 op
4107                         );
4108                         buffer_free( sql );
4109                         return NULL;
4110                 }
4111
4112                 OSRF_BUFFER_ADD( sql, query_str );
4113         }
4114
4115         if( flags & SUBCOMBO )
4116                 OSRF_BUFFER_ADD_CHAR( sql, ')' );
4117
4118         if( !(flags & SUBSELECT) )
4119                 OSRF_BUFFER_ADD_CHAR( sql, ';' );
4120
4121         return buffer_release( sql );
4122 }
4123
4124 // Translate a jsonObject into a SELECT, UNION, INTERSECT, or EXCEPT query.
4125 // The jsonObject must be a JSON_HASH with an entry for "from", "union", "intersect",
4126 // or "except" to indicate the type of query.
4127 char* buildQuery( osrfMethodContext* ctx, jsonObject* query, int flags ) {
4128         // Sanity checks
4129         if( ! query ) {
4130                 if( ctx )
4131                         osrfAppSessionStatus(
4132                                 ctx->session,
4133                                 OSRF_STATUS_INTERNALSERVERERROR,
4134                                 "osrfMethodException",
4135                                 ctx->request,
4136                                 "Malformed query; no query object"
4137                         );
4138                 osrfLogError( OSRF_LOG_MARK, "%s: Null pointer to query object", modulename );
4139                 return NULL;
4140         } else if( query->type != JSON_HASH ) {
4141                 if( ctx )
4142                         osrfAppSessionStatus(
4143                                 ctx->session,
4144                                 OSRF_STATUS_INTERNALSERVERERROR,
4145                                 "osrfMethodException",
4146                                 ctx->request,
4147                                 "Malformed query object"
4148                         );
4149                 osrfLogError(
4150                         OSRF_LOG_MARK,
4151                         "%s: Query object is %s instead of JSON_HASH",
4152                         modulename,
4153                         json_type( query->type )
4154                 );
4155                 return NULL;
4156         }
4157
4158         // Determine what kind of query it purports to be, and dispatch accordingly.
4159         if( jsonObjectGetKeyConst( query, "union" ) ||
4160                 jsonObjectGetKeyConst( query, "intersect" ) ||
4161                 jsonObjectGetKeyConst( query, "except" )) {
4162                 return doCombo( ctx, query, flags );
4163         } else {
4164                 // It is presumably a SELECT query
4165
4166                 // Push a node onto the stack for the current query.  Every level of
4167                 // subquery gets its own QueryFrame on the Stack.
4168                 push_query_frame();
4169
4170                 // Build an SQL SELECT statement
4171                 char* sql = SELECT(
4172                         ctx,
4173                         jsonObjectGetKey( query, "select" ),
4174                         jsonObjectGetKeyConst( query, "from" ),
4175                         jsonObjectGetKeyConst( query, "where" ),
4176                         jsonObjectGetKeyConst( query, "having" ),
4177                         jsonObjectGetKeyConst( query, "order_by" ),
4178                         jsonObjectGetKeyConst( query, "limit" ),
4179                         jsonObjectGetKeyConst( query, "offset" ),
4180                         flags
4181                 );
4182                 pop_query_frame();
4183                 return sql;
4184         }
4185 }
4186
4187 char* SELECT (
4188                 /* method context */ osrfMethodContext* ctx,
4189
4190                 /* SELECT   */ jsonObject* selhash,
4191                 /* FROM     */ const jsonObject* join_hash,
4192                 /* WHERE    */ const jsonObject* search_hash,
4193                 /* HAVING   */ const jsonObject* having_hash,
4194                 /* ORDER BY */ const jsonObject* order_hash,
4195                 /* LIMIT    */ const jsonObject* limit,
4196                 /* OFFSET   */ const jsonObject* offset,
4197                 /* flags    */ int flags
4198 ) {
4199         const char* locale = osrf_message_get_last_locale();
4200
4201         // general tmp objects
4202         const jsonObject* tmp_const;
4203         jsonObject* selclass = NULL;
4204         jsonObject* snode = NULL;
4205         jsonObject* onode = NULL;
4206
4207         char* string = NULL;
4208         int from_function = 0;
4209         int first = 1;
4210         int gfirst = 1;
4211         //int hfirst = 1;
4212
4213         osrfLogDebug(OSRF_LOG_MARK, "cstore SELECT locale: %s", locale ? locale : "(none)" );
4214
4215         // punt if there's no FROM clause
4216         if( !join_hash || ( join_hash->type == JSON_HASH && !join_hash->size )) {
4217                 osrfLogError(
4218                         OSRF_LOG_MARK,
4219                         "%s: FROM clause is missing or empty",
4220                         modulename
4221                 );
4222                 if( ctx )
4223                         osrfAppSessionStatus(
4224                                 ctx->session,
4225                                 OSRF_STATUS_INTERNALSERVERERROR,
4226                                 "osrfMethodException",
4227                                 ctx->request,
4228                                 "FROM clause is missing or empty in JSON query"
4229                         );
4230                 return NULL;
4231         }
4232
4233         // the core search class
4234         const char* core_class = NULL;
4235
4236         // get the core class -- the only key of the top level FROM clause, or a string
4237         if( join_hash->type == JSON_HASH ) {
4238                 jsonIterator* tmp_itr = jsonNewIterator( join_hash );
4239                 snode = jsonIteratorNext( tmp_itr );
4240
4241                 // Populate the current QueryFrame with information
4242                 // about the core class
4243                 if( add_query_core( NULL, tmp_itr->key ) ) {
4244                         if( ctx )
4245                                 osrfAppSessionStatus(
4246                                         ctx->session,
4247                                         OSRF_STATUS_INTERNALSERVERERROR,
4248                                         "osrfMethodException",
4249                                         ctx->request,
4250                                         "Unable to look up core class"
4251                                 );
4252                         return NULL;
4253                 }
4254                 core_class = curr_query->core.class_name;
4255                 join_hash = snode;
4256
4257                 jsonObject* extra = jsonIteratorNext( tmp_itr );
4258
4259                 jsonIteratorFree( tmp_itr );
4260                 snode = NULL;
4261
4262                 // There shouldn't be more than one entry in join_hash
4263                 if( extra ) {
4264                         osrfLogError(
4265                                 OSRF_LOG_MARK,
4266                                 "%s: Malformed FROM clause: extra entry in JSON_HASH",
4267                                 modulename
4268                         );
4269                         if( ctx )
4270                                 osrfAppSessionStatus(
4271                                         ctx->session,
4272                                         OSRF_STATUS_INTERNALSERVERERROR,
4273                                         "osrfMethodException",
4274                                         ctx->request,
4275                                         "Malformed FROM clause in JSON query"
4276                                 );
4277                         return NULL;    // Malformed join_hash; extra entry
4278                 }
4279         } else if( join_hash->type == JSON_ARRAY ) {
4280                 // We're selecting from a function, not from a table
4281                 from_function = 1;
4282                 core_class = jsonObjectGetString( jsonObjectGetIndex( join_hash, 0 ));
4283                 selhash = NULL;
4284
4285         } else if( join_hash->type == JSON_STRING ) {
4286                 // Populate the current QueryFrame with information
4287                 // about the core class
4288                 core_class = jsonObjectGetString( join_hash );
4289                 join_hash = NULL;
4290                 if( add_query_core( NULL, core_class ) ) {
4291                         if( ctx )
4292                                 osrfAppSessionStatus(
4293                                         ctx->session,
4294                                         OSRF_STATUS_INTERNALSERVERERROR,
4295                                         "osrfMethodException",
4296                                         ctx->request,
4297                                         "Unable to look up core class"
4298                                 );
4299                         return NULL;
4300                 }
4301         }
4302         else {
4303                 osrfLogError(
4304                         OSRF_LOG_MARK,
4305                         "%s: FROM clause is unexpected JSON type: %s",
4306                         modulename,
4307                         json_type( join_hash->type )
4308                 );
4309                 if( ctx )
4310                         osrfAppSessionStatus(
4311                                 ctx->session,
4312                                 OSRF_STATUS_INTERNALSERVERERROR,
4313                                 "osrfMethodException",
4314                                 ctx->request,
4315                                 "Ill-formed FROM clause in JSON query"
4316                         );
4317                 return NULL;
4318         }
4319
4320         // Build the join clause, if any, while filling out the list
4321         // of joined classes in the current QueryFrame.
4322         char* join_clause = NULL;
4323         if( join_hash && ! from_function ) {
4324
4325                 join_clause = searchJOIN( join_hash, &curr_query->core );
4326                 if( ! join_clause ) {
4327                         if( ctx )
4328                                 osrfAppSessionStatus(
4329                                         ctx->session,
4330                                         OSRF_STATUS_INTERNALSERVERERROR,
4331                                         "osrfMethodException",
4332                                         ctx->request,
4333                                         "Unable to construct JOIN clause(s)"
4334                                 );
4335                         return NULL;
4336                 }
4337         }
4338
4339         // For in case we don't get a select list
4340         jsonObject* defaultselhash = NULL;
4341
4342         // if there is no select list, build a default select list ...
4343         if( !selhash && !from_function ) {
4344                 jsonObject* default_list = defaultSelectList( core_class );
4345                 if( ! default_list ) {
4346                         if( ctx ) {
4347                                 osrfAppSessionStatus(
4348                                         ctx->session,
4349                                         OSRF_STATUS_INTERNALSERVERERROR,
4350                                         "osrfMethodException",
4351                                         ctx->request,
4352                                         "Unable to build default SELECT clause in JSON query"
4353                                 );
4354                                 free( join_clause );
4355                                 return NULL;
4356                         }
4357                 }
4358
4359                 selhash = defaultselhash = jsonNewObjectType( JSON_HASH );
4360                 jsonObjectSetKey( selhash, core_class, default_list );
4361         }
4362
4363         // The SELECT clause can be encoded only by a hash
4364         if( !from_function && selhash->type != JSON_HASH ) {
4365                 osrfLogError(
4366                         OSRF_LOG_MARK,
4367                         "%s: Expected JSON_HASH for SELECT clause; found %s",
4368                         modulename,
4369                         json_type( selhash->type )
4370                 );
4371
4372                 if( ctx )
4373                         osrfAppSessionStatus(
4374                                 ctx->session,
4375                                 OSRF_STATUS_INTERNALSERVERERROR,
4376                                 "osrfMethodException",
4377                                 ctx->request,
4378                                 "Malformed SELECT clause in JSON query"
4379                         );
4380                 free( join_clause );
4381                 return NULL;
4382         }
4383
4384         // If you see a null or wild card specifier for the core class, or an
4385         // empty array, replace it with a default SELECT list
4386         tmp_const = jsonObjectGetKeyConst( selhash, core_class );
4387         if( tmp_const ) {
4388                 int default_needed = 0;   // boolean
4389                 if( JSON_STRING == tmp_const->type
4390                         && !strcmp( "*", jsonObjectGetString( tmp_const ) ))
4391                                 default_needed = 1;
4392                 else if( JSON_NULL == tmp_const->type )
4393                         default_needed = 1;
4394
4395                 if( default_needed ) {
4396                         // Build a default SELECT list
4397                         jsonObject* default_list = defaultSelectList( core_class );
4398                         if( ! default_list ) {
4399                                 if( ctx ) {
4400                                         osrfAppSessionStatus(
4401                                                 ctx->session,
4402                                                 OSRF_STATUS_INTERNALSERVERERROR,
4403                                                 "osrfMethodException",
4404                                                 ctx->request,
4405                                                 "Can't build default SELECT clause in JSON query"
4406                                         );
4407                                         free( join_clause );
4408                                         return NULL;
4409                                 }
4410                         }
4411
4412                         jsonObjectSetKey( selhash, core_class, default_list );
4413                 }
4414         }
4415
4416         // temp buffers for the SELECT list and GROUP BY clause
4417         growing_buffer* select_buf = buffer_init( 128 );
4418         growing_buffer* group_buf  = buffer_init( 128 );
4419
4420         int aggregate_found = 0;     // boolean
4421
4422         // Build a select list
4423         if( from_function )   // From a function we select everything
4424                 OSRF_BUFFER_ADD_CHAR( select_buf, '*' );
4425         else {
4426
4427                 // Build the SELECT list as SQL
4428             int sel_pos = 1;
4429             first = 1;
4430             gfirst = 1;
4431             jsonIterator* selclass_itr = jsonNewIterator( selhash );
4432             while ( (selclass = jsonIteratorNext( selclass_itr )) ) {    // For each class
4433
4434                         const char* cname = selclass_itr->key;
4435
4436                         // Make sure the target relation is in the FROM clause.
4437
4438                         // At this point join_hash is a step down from the join_hash we
4439                         // received as a parameter.  If the original was a JSON_STRING,
4440                         // then json_hash is now NULL.  If the original was a JSON_HASH,
4441                         // then json_hash is now the first (and only) entry in it,
4442                         // denoting the core class.  We've already excluded the
4443                         // possibility that the original was a JSON_ARRAY, because in
4444                         // that case from_function would be non-NULL, and we wouldn't
4445                         // be here.
4446
4447                         // If the current table alias isn't in scope, bail out
4448                         ClassInfo* class_info = search_alias( cname );
4449                         if( ! class_info ) {
4450                                 osrfLogError(
4451                                         OSRF_LOG_MARK,
4452                                         "%s: SELECT clause references class not in FROM clause: \"%s\"",
4453                                         modulename,
4454                                         cname
4455                                 );
4456                                 if( ctx )
4457                                         osrfAppSessionStatus(
4458                                                 ctx->session,
4459                                                 OSRF_STATUS_INTERNALSERVERERROR,
4460                                                 "osrfMethodException",
4461                                                 ctx->request,
4462                                                 "Selected class not in FROM clause in JSON query"
4463                                         );
4464                                 jsonIteratorFree( selclass_itr );
4465                                 buffer_free( select_buf );
4466                                 buffer_free( group_buf );
4467                                 if( defaultselhash )
4468                                         jsonObjectFree( defaultselhash );
4469                                 free( join_clause );
4470                                 return NULL;
4471                         }
4472
4473                         if( selclass->type != JSON_ARRAY ) {
4474                                 osrfLogError(
4475                                         OSRF_LOG_MARK,
4476                                         "%s: Malformed SELECT list for class \"%s\"; not an array",
4477                                         modulename,
4478                                         cname
4479                                 );
4480                                 if( ctx )
4481                                         osrfAppSessionStatus(
4482                                                 ctx->session,
4483                                                 OSRF_STATUS_INTERNALSERVERERROR,
4484                                                 "osrfMethodException",
4485                                                 ctx->request,
4486                                                 "Selected class not in FROM clause in JSON query"
4487                                         );
4488
4489                                 jsonIteratorFree( selclass_itr );
4490                                 buffer_free( select_buf );
4491                                 buffer_free( group_buf );
4492                                 if( defaultselhash )
4493                                         jsonObjectFree( defaultselhash );
4494                                 free( join_clause );
4495                                 return NULL;
4496                         }
4497
4498                         // Look up some attributes of the current class
4499                         osrfHash* idlClass        = class_info->class_def;
4500                         osrfHash* class_field_set = class_info->fields;
4501                         const char* class_pkey    = osrfHashGet( idlClass, "primarykey" );
4502                         const char* class_tname   = osrfHashGet( idlClass, "tablename" );
4503
4504                         if( 0 == selclass->size ) {
4505                                 osrfLogWarning(
4506                                         OSRF_LOG_MARK,
4507                                         "%s: No columns selected from \"%s\"",
4508                                         modulename,
4509                                         cname
4510                                 );
4511                         }
4512
4513                         // stitch together the column list for the current table alias...
4514                         unsigned long field_idx = 0;
4515                         jsonObject* selfield = NULL;
4516                         while(( selfield = jsonObjectGetIndex( selclass, field_idx++ ) )) {
4517
4518                                 // If we need a separator comma, add one
4519                                 if( first ) {
4520                                         first = 0;
4521                                 } else {
4522                                         OSRF_BUFFER_ADD_CHAR( select_buf, ',' );
4523                                 }
4524
4525                                 // if the field specification is a string, add it to the list
4526                                 if( selfield->type == JSON_STRING ) {
4527
4528                                         // Look up the field in the IDL
4529                                         const char* col_name = jsonObjectGetString( selfield );
4530                                         osrfHash* field_def = NULL;
4531
4532                                         if (!osrfStringArrayContains(
4533                                                         osrfHashGet(
4534                                                                 osrfHashGet( class_field_set, col_name ),
4535                                                                 "suppress_controller"),
4536                                                         modulename
4537                                         ))
4538                                                 field_def = osrfHashGet( class_field_set, col_name );
4539
4540                                         if( !field_def ) {
4541                                                 // No such field in current class
4542                                                 osrfLogError(
4543                                                         OSRF_LOG_MARK,
4544                                                         "%s: Selected column \"%s\" not defined in IDL for class \"%s\"",
4545                                                         modulename,
4546                                                         col_name,
4547                                                         cname
4548                                                 );
4549                                                 if( ctx )
4550                                                         osrfAppSessionStatus(
4551                                                                 ctx->session,
4552                                                                 OSRF_STATUS_INTERNALSERVERERROR,
4553                                                                 "osrfMethodException",
4554                                                                 ctx->request,
4555                                                                 "Selected column not defined in JSON query"
4556                                                         );
4557                                                 jsonIteratorFree( selclass_itr );
4558                                                 buffer_free( select_buf );
4559                                                 buffer_free( group_buf );
4560                                                 if( defaultselhash )
4561                                                         jsonObjectFree( defaultselhash );
4562                                                 free( join_clause );
4563                                                 return NULL;
4564                                         } else if( str_is_true( osrfHashGet( field_def, "virtual" ) ) ) {
4565                                                 // Virtual field not allowed
4566                                                 osrfLogError(
4567                                                         OSRF_LOG_MARK,
4568                                                         "%s: Selected column \"%s\" for class \"%s\" is virtual",
4569                                                         modulename,
4570                                                         col_name,
4571                                                         cname
4572                                                 );
4573                                                 if( ctx )
4574                                                         osrfAppSessionStatus(
4575                                                                 ctx->session,
4576                                                                 OSRF_STATUS_INTERNALSERVERERROR,
4577                                                                 "osrfMethodException",
4578                                                                 ctx->request,
4579                                                                 "Selected column may not be virtual in JSON query"
4580                                                         );
4581                                                 jsonIteratorFree( selclass_itr );
4582                                                 buffer_free( select_buf );
4583                                                 buffer_free( group_buf );
4584                                                 if( defaultselhash )
4585                                                         jsonObjectFree( defaultselhash );
4586                                                 free( join_clause );
4587                                                 return NULL;
4588                                         }
4589
4590                                         if( locale ) {
4591                                                 const char* i18n;
4592                                                 if( flags & DISABLE_I18N )
4593                                                         i18n = NULL;
4594                                                 else
4595                                                         i18n = osrfHashGet( field_def, "i18n" );
4596
4597                                                 if( str_is_true( i18n ) ) {
4598                                                         buffer_fadd( select_buf, " oils_i18n_xlate('%s', '%s', '%s', "
4599                                                                 "'%s', \"%s\".%s::TEXT, '%s') AS \"%s\"",
4600                                                                 class_tname, cname, col_name, class_pkey,
4601                                                                 cname, class_pkey, locale, col_name );
4602                                                 } else {
4603                                                         buffer_fadd( select_buf, " \"%s\".%s AS \"%s\"",
4604                                                                 cname, col_name, col_name );
4605                                                 }
4606                                         } else {
4607                                                 buffer_fadd( select_buf, " \"%s\".%s AS \"%s\"",
4608                                                                 cname, col_name, col_name );
4609                                         }
4610
4611                                 // ... but it could be an object, in which case we check for a Field Transform
4612                                 } else if( selfield->type == JSON_HASH ) {
4613
4614                                         const char* col_name = jsonObjectGetString(
4615                                                         jsonObjectGetKeyConst( selfield, "column" ) );
4616
4617                                         // Get the field definition from the IDL
4618                                         osrfHash* field_def = NULL;
4619                                         if (!osrfStringArrayContains(
4620                                                         osrfHashGet(
4621                                                                 osrfHashGet( class_field_set, col_name ),
4622                                                                 "suppress_controller"),
4623                                                         modulename
4624                                         ))
4625                                                 field_def = osrfHashGet( class_field_set, col_name );
4626
4627
4628                                         if( !field_def ) {
4629                                                 // No such field in current class
4630                                                 osrfLogError(
4631                                                         OSRF_LOG_MARK,
4632                                                         "%s: Selected column \"%s\" is not defined in IDL for class \"%s\"",
4633                                                         modulename,
4634                                                         col_name,
4635                                                         cname
4636                                                 );
4637                                                 if( ctx )
4638                                                         osrfAppSessionStatus(
4639                                                                 ctx->session,
4640                                                                 OSRF_STATUS_INTERNALSERVERERROR,
4641                                                                 "osrfMethodException",
4642                                                                 ctx->request,
4643                                                                 "Selected column is not defined in JSON query"
4644                                                         );
4645                                                 jsonIteratorFree( selclass_itr );
4646                                                 buffer_free( select_buf );
4647                                                 buffer_free( group_buf );
4648                                                 if( defaultselhash )
4649                                                         jsonObjectFree( defaultselhash );
4650                                                 free( join_clause );
4651                                                 return NULL;
4652                                         } else if( str_is_true( osrfHashGet( field_def, "virtual" ))) {
4653                                                 // No such field in current class
4654                                                 osrfLogError(
4655                                                         OSRF_LOG_MARK,
4656                                                         "%s: Selected column \"%s\" is virtual for class \"%s\"",
4657                                                         modulename,
4658                                                         col_name,
4659                                                         cname
4660                                                 );
4661                                                 if( ctx )
4662                                                         osrfAppSessionStatus(
4663                                                                 ctx->session,
4664                                                                 OSRF_STATUS_INTERNALSERVERERROR,
4665                                                                 "osrfMethodException",
4666                                                                 ctx->request,
4667                                                                 "Selected column is virtual in JSON query"
4668                                                         );
4669                                                 jsonIteratorFree( selclass_itr );
4670                                                 buffer_free( select_buf );
4671                                                 buffer_free( group_buf );
4672                                                 if( defaultselhash )
4673                                                         jsonObjectFree( defaultselhash );
4674                                                 free( join_clause );
4675                                                 return NULL;
4676                                         }
4677
4678                                         // Decide what to use as a column alias
4679                                         const char* _alias;
4680                                         if((tmp_const = jsonObjectGetKeyConst( selfield, "alias" ))) {
4681                                                 _alias = jsonObjectGetString( tmp_const );
4682                                         } else if((tmp_const = jsonObjectGetKeyConst( selfield, "result_field" ))) { // Use result_field name as the alias
4683                                                 _alias = jsonObjectGetString( tmp_const );
4684                                         } else {         // Use field name as the alias
4685                                                 _alias = col_name;
4686                                         }
4687
4688                                         if( jsonObjectGetKeyConst( selfield, "transform" )) {
4689                                                 char* transform_str = searchFieldTransform(
4690                                                         class_info->alias, field_def, selfield );
4691                                                 if( transform_str ) {
4692                                                         buffer_fadd( select_buf, " %s AS \"%s\"", transform_str, _alias );
4693                                                         free( transform_str );
4694                                                 } else {
4695                                                         if( ctx )
4696                                                                 osrfAppSessionStatus(
4697                                                                         ctx->session,
4698                                                                         OSRF_STATUS_INTERNALSERVERERROR,
4699                                                                         "osrfMethodException",
4700                                                                         ctx->request,
4701                                                                         "Unable to generate transform function in JSON query"
4702                                                                 );
4703                                                         jsonIteratorFree( selclass_itr );
4704                                                         buffer_free( select_buf );
4705                                                         buffer_free( group_buf );
4706                                                         if( defaultselhash )
4707                                                                 jsonObjectFree( defaultselhash );
4708                                                         free( join_clause );
4709                                                         return NULL;
4710                                                 }
4711                                         } else {
4712
4713                                                 if( locale ) {
4714                                                         const char* i18n;
4715                                                         if( flags & DISABLE_I18N )
4716                                                                 i18n = NULL;
4717                                                         else
4718                                                                 i18n = osrfHashGet( field_def, "i18n" );
4719
4720                                                         if( str_is_true( i18n ) ) {
4721                                                                 buffer_fadd( select_buf,
4722                                                                         " oils_i18n_xlate('%s', '%s', '%s', '%s', "
4723                                                                         "\"%s\".%s::TEXT, '%s') AS \"%s\"",
4724                                                                         class_tname, cname, col_name, class_pkey, cname,
4725                                                                         class_pkey, locale, _alias );
4726                                                         } else {
4727                                                                 buffer_fadd( select_buf, " \"%s\".%s AS \"%s\"",
4728                                                                         cname, col_name, _alias );
4729                                                         }
4730                                                 } else {
4731                                                         buffer_fadd( select_buf, " \"%s\".%s AS \"%s\"",
4732                                                                 cname, col_name, _alias );
4733                                                 }
4734                                         }
4735                                 }
4736                                 else {
4737                                         osrfLogError(
4738                                                 OSRF_LOG_MARK,
4739                                                 "%s: Selected item is unexpected JSON type: %s",
4740                                                 modulename,
4741                                                 json_type( selfield->type )
4742                                         );
4743                                         if( ctx )
4744                                                 osrfAppSessionStatus(
4745                                                         ctx->session,
4746                                                         OSRF_STATUS_INTERNALSERVERERROR,
4747                                                         "osrfMethodException",
4748                                                         ctx->request,
4749                                                         "Ill-formed SELECT item in JSON query"
4750                                                 );
4751                                         jsonIteratorFree( selclass_itr );
4752                                         buffer_free( select_buf );
4753                                         buffer_free( group_buf );
4754                                         if( defaultselhash )
4755                                                 jsonObjectFree( defaultselhash );
4756                                         free( join_clause );
4757                                         return NULL;
4758                                 }
4759
4760                                 const jsonObject* agg_obj = jsonObjectGetKeyConst( selfield, "aggregate" );
4761                                 if( obj_is_true( agg_obj ) )
4762                                         aggregate_found = 1;
4763                                 else {
4764                                         // Append a comma (except for the first one)
4765                                         // and add the column to a GROUP BY clause
4766                                         if( gfirst )
4767                                                 gfirst = 0;
4768                                         else
4769                                                 OSRF_BUFFER_ADD_CHAR( group_buf, ',' );
4770
4771                                         buffer_fadd( group_buf, " %d", sel_pos );
4772                                 }
4773
4774 #if 0
4775                             if (is_agg->size || (flags & SELECT_DISTINCT)) {
4776
4777                                         const jsonObject* aggregate_obj = jsonObjectGetKeyConst( selfield, "aggregate");
4778                                     if ( ! obj_is_true( aggregate_obj ) ) {
4779                                             if (gfirst) {
4780                                                     gfirst = 0;
4781                                             } else {
4782                                                         OSRF_BUFFER_ADD_CHAR( group_buf, ',' );
4783                                             }
4784
4785                                             buffer_fadd(group_buf, " %d", sel_pos);
4786
4787                                         /*
4788                                     } else if (is_agg = jsonObjectGetKeyConst( selfield, "having" )) {
4789                                             if (gfirst) {
4790                                                     gfirst = 0;
4791                                             } else {
4792                                                         OSRF_BUFFER_ADD_CHAR( group_buf, ',' );
4793                                             }
4794
4795                                             _column = searchFieldTransform(class_info->alias, field, selfield);
4796                                                 OSRF_BUFFER_ADD_CHAR(group_buf, ' ');
4797                                                 OSRF_BUFFER_ADD(group_buf, _column);
4798                                             _column = searchFieldTransform(class_info->alias, field, selfield);
4799                                         */
4800                                     }
4801                             }
4802 #endif
4803
4804                                 sel_pos++;
4805                         } // end while -- iterating across SELECT columns
4806
4807                 } // end while -- iterating across classes
4808
4809                 jsonIteratorFree( selclass_itr );
4810         }
4811
4812         char* col_list = buffer_release( select_buf );
4813
4814         // Make sure the SELECT list isn't empty.  This can happen, for example,
4815         // if we try to build a default SELECT clause from a non-core table.
4816
4817         if( ! *col_list ) {
4818                 osrfLogError( OSRF_LOG_MARK, "%s: SELECT clause is empty", modulename );
4819                 if( ctx )
4820                         osrfAppSessionStatus(
4821                                 ctx->session,
4822                                 OSRF_STATUS_INTERNALSERVERERROR,
4823                                 "osrfMethodException",
4824                                 ctx->request,
4825                                 "SELECT list is empty"
4826                 );
4827                 free( col_list );
4828                 buffer_free( group_buf );
4829                 if( defaultselhash )
4830                         jsonObjectFree( defaultselhash );
4831                 free( join_clause );
4832                 return NULL;
4833         }
4834
4835         char* table = NULL;
4836         if( from_function )
4837                 table = searchValueTransform( join_hash );
4838         else
4839                 table = strdup( curr_query->core.source_def );
4840
4841         if( !table ) {
4842                 if( ctx )
4843                         osrfAppSessionStatus(
4844                                 ctx->session,
4845                                 OSRF_STATUS_INTERNALSERVERERROR,
4846                                 "osrfMethodException",
4847                                 ctx->request,
4848                                 "Unable to identify table for core class"
4849                         );
4850                 free( col_list );
4851                 buffer_free( group_buf );
4852                 if( defaultselhash )
4853                         jsonObjectFree( defaultselhash );
4854                 free( join_clause );
4855                 return NULL;
4856         }
4857
4858         // Put it all together
4859         growing_buffer* sql_buf = buffer_init( 128 );
4860         buffer_fadd(sql_buf, "SELECT %s FROM %s AS \"%s\" ", col_list, table, core_class );
4861         free( col_list );
4862         free( table );
4863
4864         // Append the join clause, if any
4865         if( join_clause ) {
4866                 buffer_add(sql_buf, join_clause );
4867                 free( join_clause );
4868         }
4869
4870         char* order_by_list = NULL;
4871         char* having_buf = NULL;
4872
4873         if( !from_function ) {
4874
4875                 // Build a WHERE clause, if there is one
4876                 if( search_hash ) {
4877                         buffer_add( sql_buf, " WHERE " );
4878
4879                         // and it's on the WHERE clause
4880                         char* pred = searchWHERE( search_hash, &curr_query->core, AND_OP_JOIN, ctx );
4881                         if( ! pred ) {
4882                                 if( ctx ) {
4883                                         osrfAppSessionStatus(
4884                                                 ctx->session,
4885                                                 OSRF_STATUS_INTERNALSERVERERROR,
4886                                                 "osrfMethodException",
4887                                                 ctx->request,
4888                                                 "Severe query error in WHERE predicate -- see error log for more details"
4889                                         );
4890                                 }
4891                                 buffer_free( group_buf );
4892                                 buffer_free( sql_buf );
4893                                 if( defaultselhash )
4894                                         jsonObjectFree( defaultselhash );
4895                                 return NULL;
4896                         }
4897
4898                         buffer_add( sql_buf, pred );
4899                         free( pred );
4900                 }
4901
4902                 // Build a HAVING clause, if there is one
4903                 if( having_hash ) {
4904
4905                         // and it's on the the WHERE clause
4906                         having_buf = searchWHERE( having_hash, &curr_query->core, AND_OP_JOIN, ctx );
4907
4908                         if( ! having_buf ) {
4909                                 if( ctx ) {
4910                                                 osrfAppSessionStatus(
4911                                                 ctx->session,
4912                                                 OSRF_STATUS_INTERNALSERVERERROR,
4913                                                 "osrfMethodException",
4914                                                 ctx->request,
4915                                                 "Severe query error in HAVING predicate -- see error log for more details"
4916                                         );
4917                                 }
4918                                 buffer_free( group_buf );
4919                                 buffer_free( sql_buf );
4920                                 if( defaultselhash )
4921                                         jsonObjectFree( defaultselhash );
4922                                 return NULL;
4923                         }
4924                 }
4925
4926                 // Build an ORDER BY clause, if there is one
4927                 if( NULL == order_hash )
4928                         ;  // No ORDER BY? do nothing
4929                 else if( JSON_ARRAY == order_hash->type ) {
4930                         order_by_list = buildOrderByFromArray( ctx, order_hash );
4931                         if( !order_by_list ) {
4932                                 free( having_buf );
4933                                 buffer_free( group_buf );
4934                                 buffer_free( sql_buf );
4935                                 if( defaultselhash )
4936                                         jsonObjectFree( defaultselhash );
4937                                 return NULL;
4938                         }
4939                 } else if( JSON_HASH == order_hash->type ) {
4940                         // This hash is keyed on class alias.  Each class has either
4941                         // an array of field names or a hash keyed on field name.
4942                         growing_buffer* order_buf = NULL;  // to collect ORDER BY list
4943                         jsonIterator* class_itr = jsonNewIterator( order_hash );
4944                         while( (snode = jsonIteratorNext( class_itr )) ) {
4945
4946                                 ClassInfo* order_class_info = search_alias( class_itr->key );
4947                                 if( ! order_class_info ) {
4948                                         osrfLogError( OSRF_LOG_MARK,
4949                                                 "%s: Invalid class \"%s\" referenced in ORDER BY clause",
4950                                                 modulename, class_itr->key );
4951                                         if( ctx )
4952                                                 osrfAppSessionStatus(
4953                                                         ctx->session,
4954                                                         OSRF_STATUS_INTERNALSERVERERROR,
4955                                                         "osrfMethodException",
4956                                                         ctx->request,
4957                                                         "Invalid class referenced in ORDER BY clause -- "
4958                                                                 "see error log for more details"
4959                                                 );
4960                                         jsonIteratorFree( class_itr );
4961                                         buffer_free( order_buf );
4962                                         free( having_buf );
4963                                         buffer_free( group_buf );
4964                                         buffer_free( sql_buf );
4965                                         if( defaultselhash )
4966                                                 jsonObjectFree( defaultselhash );
4967                                         return NULL;
4968                                 }
4969
4970                                 osrfHash* field_list_def = order_class_info->fields;
4971
4972                                 if( snode->type == JSON_HASH ) {
4973
4974                                         // Hash is keyed on field names from the current class.  For each field
4975                                         // there is another layer of hash to define the sorting details, if any,
4976                                         // or a string to indicate direction of sorting.
4977                                         jsonIterator* order_itr = jsonNewIterator( snode );
4978                                         while( (onode = jsonIteratorNext( order_itr )) ) {
4979
4980                                                 osrfHash* field_def = osrfHashGet( field_list_def, order_itr->key );
4981                                                 if( !field_def ) {
4982                                                         osrfLogError( OSRF_LOG_MARK,
4983                                                                 "%s: Invalid field \"%s\" in ORDER BY clause",
4984                                                                 modulename, order_itr->key );
4985                                                         if( ctx )
4986                                                                 osrfAppSessionStatus(
4987                                                                         ctx->session,
4988                                                                         OSRF_STATUS_INTERNALSERVERERROR,
4989                                                                         "osrfMethodException",
4990                                                                         ctx->request,
4991                                                                         "Invalid field in ORDER BY clause -- "
4992                                                                         "see error log for more details"
4993                                                                 );
4994                                                         jsonIteratorFree( order_itr );
4995                                                         jsonIteratorFree( class_itr );
4996                                                         buffer_free( order_buf );
4997                                                         free( having_buf );
4998                                                         buffer_free( group_buf );
4999                                                         buffer_free( sql_buf );
5000                                                         if( defaultselhash )
5001                                                                 jsonObjectFree( defaultselhash );
5002                                                         return NULL;
5003                                                 } else if( str_is_true( osrfHashGet( field_def, "virtual" ) ) ) {
5004                                                         osrfLogError( OSRF_LOG_MARK,
5005                                                                 "%s: Virtual field \"%s\" in ORDER BY clause",
5006                                                                 modulename, order_itr->key );
5007                                                         if( ctx )
5008                                                                 osrfAppSessionStatus(
5009                                                                         ctx->session,
5010                                                                         OSRF_STATUS_INTERNALSERVERERROR,
5011                                                                         "osrfMethodException",
5012                                                                         ctx->request,
5013                                                                         "Virtual field in ORDER BY clause -- "
5014                                                                         "see error log for more details"
5015                                                         );
5016                                                         jsonIteratorFree( order_itr );
5017                                                         jsonIteratorFree( class_itr );
5018                                                         buffer_free( order_buf );
5019                                                         free( having_buf );
5020                                                         buffer_free( group_buf );
5021                                                         buffer_free( sql_buf );
5022                                                         if( defaultselhash )
5023                                                                 jsonObjectFree( defaultselhash );
5024                                                         return NULL;
5025                                                 }
5026
5027                                                 const char* direction = NULL;
5028                                                 if( onode->type == JSON_HASH ) {
5029                                                         if( jsonObjectGetKeyConst( onode, "transform" ) ) {
5030                                                                 string = searchFieldTransform(
5031                                                                         class_itr->key,
5032                                                                         osrfHashGet( field_list_def, order_itr->key ),
5033                                                                         onode
5034                                                                 );
5035                                                                 if( ! string ) {
5036                                                                         if( ctx ) osrfAppSessionStatus(
5037                                                                                 ctx->session,
5038                                                                                 OSRF_STATUS_INTERNALSERVERERROR,
5039                                                                                 "osrfMethodException",
5040                                                                                 ctx->request,
5041                                                                                 "Severe query error in ORDER BY clause -- "
5042                                                                                 "see error log for more details"
5043                                                                         );
5044                                                                         jsonIteratorFree( order_itr );
5045                                                                         jsonIteratorFree( class_itr );
5046                                                                         free( having_buf );
5047                                                                         buffer_free( group_buf );
5048                                                                         buffer_free( order_buf);
5049                                                                         buffer_free( sql_buf );
5050                                                                         if( defaultselhash )
5051                                                                                 jsonObjectFree( defaultselhash );
5052                                                                         return NULL;
5053                                                                 }
5054                                                         } else {
5055                                                                 growing_buffer* field_buf = buffer_init( 16 );
5056                                                                 buffer_fadd( field_buf, "\"%s\".%s",
5057                                                                         class_itr->key, order_itr->key );
5058                                                                 string = buffer_release( field_buf );
5059                                                         }
5060
5061                                                         if( (tmp_const = jsonObjectGetKeyConst( onode, "direction" )) ) {
5062                                                                 const char* dir = jsonObjectGetString( tmp_const );
5063                                                                 if(!strncasecmp( dir, "d", 1 )) {
5064                                                                         direction = " DESC";
5065                                                                 } else {
5066                                                                         direction = " ASC";
5067                                                                 }
5068                                                         }
5069
5070                                                 } else if( JSON_NULL == onode->type || JSON_ARRAY == onode->type ) {
5071                                                         osrfLogError( OSRF_LOG_MARK,
5072                                                                 "%s: Expected JSON_STRING in ORDER BY clause; found %s",
5073                                                                 modulename, json_type( onode->type ) );
5074                                                         if( ctx )
5075                                                                 osrfAppSessionStatus(
5076                                                                         ctx->session,
5077                                                                         OSRF_STATUS_INTERNALSERVERERROR,
5078                                                                         "osrfMethodException",
5079                                                                         ctx->request,
5080                                                                         "Malformed ORDER BY clause -- see error log for more details"
5081                                                                 );
5082                                                         jsonIteratorFree( order_itr );
5083                                                         jsonIteratorFree( class_itr );
5084                                                         free( having_buf );
5085                                                         buffer_free( group_buf );
5086                                                         buffer_free( order_buf );
5087                                                         buffer_free( sql_buf );
5088                                                         if( defaultselhash )
5089                                                                 jsonObjectFree( defaultselhash );
5090                                                         return NULL;
5091
5092                                                 } else {
5093                                                         string = strdup( order_itr->key );
5094                                                         const char* dir = jsonObjectGetString( onode );
5095                                                         if( !strncasecmp( dir, "d", 1 )) {
5096                                                                 direction = " DESC";
5097                                                         } else {
5098                                                                 direction = " ASC";
5099                                                         }
5100                                                 }
5101
5102                                                 if( order_buf )
5103                                                         OSRF_BUFFER_ADD( order_buf, ", " );
5104                                                 else
5105                                                         order_buf = buffer_init( 128 );
5106
5107                                                 OSRF_BUFFER_ADD( order_buf, string );
5108                                                 free( string );
5109
5110                                                 if( direction ) {
5111                                                          OSRF_BUFFER_ADD( order_buf, direction );
5112                                                 }
5113
5114                                         } // end while
5115                                         jsonIteratorFree( order_itr );
5116
5117                                 } else if( snode->type == JSON_ARRAY ) {
5118
5119                                         // Array is a list of fields from the current class
5120                                         unsigned long order_idx = 0;
5121                                         while(( onode = jsonObjectGetIndex( snode, order_idx++ ) )) {
5122
5123                                                 const char* _f = jsonObjectGetString( onode );
5124
5125                                                 osrfHash* field_def = osrfHashGet( field_list_def, _f );
5126                                                 if( !field_def ) {
5127                                                         osrfLogError( OSRF_LOG_MARK,
5128                                                                         "%s: Invalid field \"%s\" in ORDER BY clause",
5129                                                                         modulename, _f );
5130                                                         if( ctx )
5131                                                                 osrfAppSessionStatus(
5132                                                                         ctx->session,
5133                                                                         OSRF_STATUS_INTERNALSERVERERROR,
5134                                                                         "osrfMethodException",
5135                                                                         ctx->request,
5136                                                                         "Invalid field in ORDER BY clause -- "
5137                                                                         "see error log for more details"
5138                                                                 );
5139                                                         jsonIteratorFree( class_itr );
5140                                                         buffer_free( order_buf );
5141                                                         free( having_buf );
5142                                                         buffer_free( group_buf );
5143                                                         buffer_free( sql_buf );
5144                                                         if( defaultselhash )
5145                                                                 jsonObjectFree( defaultselhash );
5146                                                         return NULL;
5147                                                 } else if( str_is_true( osrfHashGet( field_def, "virtual" ) ) ) {
5148                                                         osrfLogError( OSRF_LOG_MARK,
5149                                                                 "%s: Virtual field \"%s\" in ORDER BY clause",
5150                                                                 modulename, _f );
5151                                                         if( ctx )
5152                                                                 osrfAppSessionStatus(
5153                                                                         ctx->session,
5154                                                                         OSRF_STATUS_INTERNALSERVERERROR,
5155                                                                         "osrfMethodException",
5156                                                                         ctx->request,
5157                                                                         "Virtual field in ORDER BY clause -- "
5158                                                                         "see error log for more details"
5159                                                                 );
5160                                                         jsonIteratorFree( class_itr );
5161                                                         buffer_free( order_buf );
5162                                                         free( having_buf );
5163                                                         buffer_free( group_buf );
5164                                                         buffer_free( sql_buf );
5165                                                         if( defaultselhash )
5166                                                                 jsonObjectFree( defaultselhash );
5167                                                         return NULL;
5168                                                 }
5169
5170                                                 if( order_buf )
5171                                                         OSRF_BUFFER_ADD( order_buf, ", " );
5172                                                 else
5173                                                         order_buf = buffer_init( 128 );
5174
5175                                                 buffer_fadd( order_buf, "\"%s\".%s", class_itr->key, _f );
5176
5177                                         } // end while
5178
5179                                 // IT'S THE OOOOOOOOOOOLD STYLE!
5180                                 } else {
5181                                         osrfLogError( OSRF_LOG_MARK,
5182                                                 "%s: Possible SQL injection attempt; direct order by is not allowed",
5183                                                 modulename );
5184                                         if(ctx) {
5185                                                 osrfAppSessionStatus(
5186                                                         ctx->session,
5187                                                         OSRF_STATUS_INTERNALSERVERERROR,
5188                                                         "osrfMethodException",
5189                                                         ctx->request,
5190                                                         "Severe query error -- see error log for more details"
5191                                                 );
5192                                         }
5193
5194                                         free( having_buf );
5195                                         buffer_free( group_buf );
5196                                         buffer_free( order_buf );
5197                                         buffer_free( sql_buf );
5198                                         if( defaultselhash )
5199                                                 jsonObjectFree( defaultselhash );
5200                                         jsonIteratorFree( class_itr );
5201                                         return NULL;
5202                                 }
5203                         } // end while
5204                         jsonIteratorFree( class_itr );
5205                         if( order_buf )
5206                                 order_by_list = buffer_release( order_buf );
5207                 } else {
5208                         osrfLogError( OSRF_LOG_MARK,
5209                                 "%s: Malformed ORDER BY clause; expected JSON_HASH or JSON_ARRAY, found %s",
5210                                 modulename, json_type( order_hash->type ) );
5211                         if( ctx )
5212                                 osrfAppSessionStatus(
5213                                         ctx->session,
5214                                         OSRF_STATUS_INTERNALSERVERERROR,
5215                                         "osrfMethodException",
5216                                         ctx->request,
5217                                         "Malformed ORDER BY clause -- see error log for more details"
5218                                 );
5219                         free( having_buf );
5220                         buffer_free( group_buf );
5221                         buffer_free( sql_buf );
5222                         if( defaultselhash )
5223                                 jsonObjectFree( defaultselhash );
5224                         return NULL;
5225                 }
5226         }
5227
5228         string = buffer_release( group_buf );
5229
5230         if( *string && ( aggregate_found || (flags & SELECT_DISTINCT) ) ) {
5231                 OSRF_BUFFER_ADD( sql_buf, " GROUP BY " );
5232                 OSRF_BUFFER_ADD( sql_buf, string );
5233         }
5234
5235         free( string );
5236
5237         if( having_buf && *having_buf ) {
5238                 OSRF_BUFFER_ADD( sql_buf, " HAVING " );
5239                 OSRF_BUFFER_ADD( sql_buf, having_buf );
5240                 free( having_buf );
5241         }
5242
5243         if( order_by_list ) {
5244
5245                 if( *order_by_list ) {
5246                         OSRF_BUFFER_ADD( sql_buf, " ORDER BY " );
5247                         OSRF_BUFFER_ADD( sql_buf, order_by_list );
5248                 }
5249
5250                 free( order_by_list );
5251         }
5252
5253         if( limit ){
5254                 const char* str = jsonObjectGetString( limit );
5255                 if (str) { // limit could be JSON_NULL, etc.
5256                         buffer_fadd( sql_buf, " LIMIT %d", atoi( str ));
5257                 }
5258         }
5259
5260         if( offset ) {
5261                 const char* str = jsonObjectGetString( offset );
5262                 if (str) {
5263                         buffer_fadd( sql_buf, " OFFSET %d", atoi( str ));
5264                 }
5265         }
5266
5267         if( !(flags & SUBSELECT) )
5268                 OSRF_BUFFER_ADD_CHAR( sql_buf, ';' );
5269
5270         if( defaultselhash )
5271                  jsonObjectFree( defaultselhash );
5272
5273         return buffer_release( sql_buf );
5274
5275 } // end of SELECT()
5276
5277 /**
5278         @brief Build a list of ORDER BY expressions.
5279         @param ctx Pointer to the method context.
5280         @param order_array Pointer to a JSON_ARRAY of field specifications.
5281         @return Pointer to a string containing a comma-separated list of ORDER BY expressions.
5282         Each expression may be either a column reference or a function call whose first parameter
5283         is a column reference.
5284
5285         Each entry in @a order_array must be a JSON_HASH with values for "class" and "field".
5286         It may optionally include entries for "direction" and/or "transform".
5287
5288         The calling code is responsible for freeing the returned string.
5289 */
5290 static char* buildOrderByFromArray( osrfMethodContext* ctx, const jsonObject* order_array ) {
5291         if( ! order_array ) {
5292                 osrfLogError( OSRF_LOG_MARK, "%s: Logic error: NULL pointer for ORDER BY clause",
5293                         modulename );
5294                 if( ctx )
5295                         osrfAppSessionStatus(
5296                                 ctx->session,
5297                                 OSRF_STATUS_INTERNALSERVERERROR,
5298                                 "osrfMethodException",
5299                                 ctx->request,
5300                                 "Logic error: ORDER BY clause expected, not found; "
5301                                         "see error log for more details"
5302                         );
5303                 return NULL;
5304         } else if( order_array->type != JSON_ARRAY ) {
5305                 osrfLogError( OSRF_LOG_MARK,
5306                         "%s: Logic error: Expected JSON_ARRAY for ORDER BY clause, not found", modulename );
5307                 if( ctx )
5308                         osrfAppSessionStatus(
5309                         ctx->session,
5310                         OSRF_STATUS_INTERNALSERVERERROR,
5311                         "osrfMethodException",
5312                         ctx->request,
5313                         "Logic error: Unexpected format for ORDER BY clause; see error log for more details" );
5314                 return NULL;
5315         }
5316
5317         growing_buffer* order_buf = buffer_init( 128 );
5318         int first = 1;        // boolean
5319         int order_idx = 0;
5320         jsonObject* order_spec;
5321         while( (order_spec = jsonObjectGetIndex( order_array, order_idx++ ))) {
5322
5323                 if( JSON_HASH != order_spec->type ) {
5324                         osrfLogError( OSRF_LOG_MARK,
5325                                 "%s: Malformed field specification in ORDER BY clause; "
5326                                 "expected JSON_HASH, found %s",
5327                                 modulename, json_type( order_spec->type ) );
5328                         if( ctx )
5329                                 osrfAppSessionStatus(
5330                                          ctx->session,
5331                                         OSRF_STATUS_INTERNALSERVERERROR,
5332                                         "osrfMethodException",
5333                                         ctx->request,
5334                                         "Malformed ORDER BY clause -- see error log for more details"
5335                                 );
5336                         buffer_free( order_buf );
5337                         return NULL;
5338                 }
5339
5340                 const char* class_alias =
5341                         jsonObjectGetString( jsonObjectGetKeyConst( order_spec, "class" ));
5342                 const char* field =
5343                         jsonObjectGetString( jsonObjectGetKeyConst( order_spec, "field" ));
5344
5345                 jsonObject* compare_to = jsonObjectGetKey( order_spec, "compare" );
5346
5347                 if( !field || !class_alias ) {
5348                         osrfLogError( OSRF_LOG_MARK,
5349                                 "%s: Missing class or field name in field specification of ORDER BY clause",
5350                                 modulename );
5351                         if( ctx )
5352                                 osrfAppSessionStatus(
5353                                         ctx->session,
5354                                         OSRF_STATUS_INTERNALSERVERERROR,
5355                                         "osrfMethodException",
5356                                         ctx->request,
5357                                         "Malformed ORDER BY clause -- see error log for more details"
5358                                 );
5359                         buffer_free( order_buf );
5360                         return NULL;
5361                 }
5362
5363                 const ClassInfo* order_class_info = search_alias( class_alias );
5364                 if( ! order_class_info ) {
5365                         osrfLogInternal( OSRF_LOG_MARK, "%s: ORDER BY clause references class \"%s\" "
5366                                 "not in FROM clause, skipping it", modulename, class_alias );
5367                         continue;
5368                 }
5369
5370                 // Add a separating comma, except at the beginning
5371                 if( first )
5372                         first = 0;
5373                 else
5374                         OSRF_BUFFER_ADD( order_buf, ", " );
5375
5376                 osrfHash* field_def = osrfHashGet( order_class_info->fields, field );
5377                 if( !field_def ) {
5378                         osrfLogError( OSRF_LOG_MARK,
5379                                 "%s: Invalid field \"%s\".%s referenced in ORDER BY clause",
5380                                 modulename, class_alias, field );
5381                         if( ctx )
5382                                 osrfAppSessionStatus(
5383                                         ctx->session,
5384                                         OSRF_STATUS_INTERNALSERVERERROR,
5385                                         "osrfMethodException",
5386                                         ctx->request,
5387                                         "Invalid field referenced in ORDER BY clause -- "
5388                                         "see error log for more details"
5389                                 );
5390                         free( order_buf );
5391                         return NULL;
5392                 } else if( str_is_true( osrfHashGet( field_def, "virtual" ) ) ) {
5393                         osrfLogError( OSRF_LOG_MARK, "%s: Virtual field \"%s\" in ORDER BY clause",
5394                                 modulename, field );
5395                         if( ctx )
5396                                 osrfAppSessionStatus(
5397                                         ctx->session,
5398                                         OSRF_STATUS_INTERNALSERVERERROR,
5399                                         "osrfMethodException",
5400                                         ctx->request,
5401                                         "Virtual field in ORDER BY clause -- see error log for more details"
5402                                 );
5403                         buffer_free( order_buf );
5404                         return NULL;
5405                 }
5406
5407                 if( jsonObjectGetKeyConst( order_spec, "transform" )) {
5408                         char* transform_str = searchFieldTransform( class_alias, field_def, order_spec );
5409                         if( ! transform_str ) {
5410                                 if( ctx )
5411                                         osrfAppSessionStatus(
5412                                                 ctx->session,
5413                                                 OSRF_STATUS_INTERNALSERVERERROR,
5414                                                 "osrfMethodException",
5415                                                 ctx->request,
5416                                                 "Severe query error in ORDER BY clause -- "
5417                                                 "see error log for more details"
5418                                         );
5419                                 buffer_free( order_buf );
5420                                 return NULL;
5421                         }
5422
5423                         OSRF_BUFFER_ADD( order_buf, transform_str );
5424                         free( transform_str );
5425                 } else if( compare_to ) {
5426                         char* compare_str = searchPredicate( order_class_info, field_def, compare_to, ctx );
5427                         if( ! compare_str ) {
5428                                 if( ctx )
5429                                         osrfAppSessionStatus(
5430                                                 ctx->session,
5431                                                 OSRF_STATUS_INTERNALSERVERERROR,
5432                                                 "osrfMethodException",
5433                                                 ctx->request,
5434                                                 "Severe query error in ORDER BY clause -- "
5435                                                 "see error log for more details"
5436                                         );
5437                                 buffer_free( order_buf );
5438                                 return NULL;
5439                         }
5440
5441                         buffer_fadd( order_buf, "(%s)", compare_str );
5442                         free( compare_str );
5443                 }
5444                 else
5445                         buffer_fadd( order_buf, "\"%s\".%s", class_alias, field );
5446
5447                 const char* direction =
5448                         jsonObjectGetString( jsonObjectGetKeyConst( order_spec, "direction" ) );
5449                 if( direction ) {
5450                         if( direction[ 0 ] && ( 'd' == direction[ 0 ] || 'D' == direction[ 0 ] ) )
5451                                 OSRF_BUFFER_ADD( order_buf, " DESC" );
5452                         else
5453                                 OSRF_BUFFER_ADD( order_buf, " ASC" );
5454                 }
5455         }
5456
5457         return buffer_release( order_buf );
5458 }
5459
5460 /**
5461         @brief Build a SELECT statement.
5462         @param search_hash Pointer to a JSON_HASH or JSON_ARRAY encoding the WHERE clause.
5463         @param rest_of_query Pointer to a JSON_HASH containing any other SQL clauses.
5464         @param meta Pointer to the class metadata for the core class.
5465         @param ctx Pointer to the method context.
5466         @return Pointer to a character string containing the WHERE clause; or NULL upon error.
5467
5468         Within the rest_of_query hash, the meaningful keys are "join", "select", "no_i18n",
5469         "order_by", "limit", and "offset".
5470
5471         The SELECT statements built here are distinct from those built for the json_query method.
5472 */
5473 static char* buildSELECT ( const jsonObject* search_hash, jsonObject* rest_of_query,
5474         osrfHash* meta, osrfMethodContext* ctx ) {
5475
5476         const char* locale = osrf_message_get_last_locale();
5477
5478         osrfHash* fields = osrfHashGet( meta, "fields" );
5479         const char* core_class = osrfHashGet( meta, "classname" );
5480
5481         const jsonObject* join_hash = jsonObjectGetKeyConst( rest_of_query, "join" );
5482
5483         jsonObject* selhash = NULL;
5484         jsonObject* defaultselhash = NULL;
5485
5486         growing_buffer* sql_buf = buffer_init( 128 );
5487         growing_buffer* select_buf = buffer_init( 128 );
5488
5489         if( !(selhash = jsonObjectGetKey( rest_of_query, "select" )) ) {
5490                 defaultselhash = jsonNewObjectType( JSON_HASH );
5491                 selhash = defaultselhash;
5492         }
5493
5494         // If there's no SELECT list for the core class, build one
5495         if( !jsonObjectGetKeyConst( selhash, core_class ) ) {
5496                 jsonObject* field_list = jsonNewObjectType( JSON_ARRAY );
5497
5498                 // Add every non-virtual field to the field list
5499                 osrfHash* field_def = NULL;
5500                 osrfHashIterator* field_itr = osrfNewHashIterator( fields );
5501                 while( ( field_def = osrfHashIteratorNext( field_itr ) ) ) {
5502                         if( ! str_is_true( osrfHashGet( field_def, "virtual" ) ) ) {
5503                                 const char* field = osrfHashIteratorKey( field_itr );
5504                                 jsonObjectPush( field_list, jsonNewObject( field ) );
5505                         }
5506                 }
5507                 osrfHashIteratorFree( field_itr );
5508                 jsonObjectSetKey( selhash, core_class, field_list );
5509         }
5510
5511         // Build a list of columns for the SELECT clause
5512         int first = 1;
5513         const jsonObject* snode = NULL;
5514         jsonIterator* class_itr = jsonNewIterator( selhash );
5515         while( (snode = jsonIteratorNext( class_itr )) ) {        // For each class
5516
5517                 // If the class isn't in the IDL, ignore it
5518                 const char* cname = class_itr->key;
5519                 osrfHash* idlClass = osrfHashGet( oilsIDL(), cname );
5520                 if( !idlClass )
5521                         continue;
5522
5523                 // If the class isn't the core class, and isn't in the JOIN clause, ignore it
5524                 if( strcmp( core_class, class_itr->key )) {
5525                         if( !join_hash )
5526                                 continue;
5527
5528                         jsonObject* found = jsonObjectFindPath( join_hash, "//%s", class_itr->key );
5529                         if( !found->size ) {
5530                                 jsonObjectFree( found );
5531                                 continue;
5532                         }
5533
5534                         jsonObjectFree( found );
5535                 }
5536
5537                 const jsonObject* node = NULL;
5538                 jsonIterator* select_itr = jsonNewIterator( snode );
5539                 while( (node = jsonIteratorNext( select_itr )) ) {
5540                         const char* item_str = jsonObjectGetString( node );
5541                         osrfHash* field = osrfHashGet( osrfHashGet( idlClass, "fields" ), item_str );
5542                         char* fname = osrfHashGet( field, "name" );
5543
5544                         if( !field )
5545                                 continue;
5546
5547                         if (osrfStringArrayContains( osrfHashGet(field, "suppress_controller"), modulename ))
5548                                 continue;
5549
5550                         if( first ) {
5551                                 first = 0;
5552                         } else {
5553                                 OSRF_BUFFER_ADD_CHAR( select_buf, ',' );
5554                         }
5555
5556                         if( locale ) {
5557                                 const char* i18n;
5558                                 const jsonObject* no_i18n_obj = jsonObjectGetKeyConst( rest_of_query, "no_i18n" );
5559                                 if( obj_is_true( no_i18n_obj ) )    // Suppress internationalization?
5560                                         i18n = NULL;
5561                                 else
5562                                         i18n = osrfHashGet( field, "i18n" );
5563
5564                                 if( str_is_true( i18n ) ) {
5565                                         char* pkey = osrfHashGet( idlClass, "primarykey" );
5566                                         char* tname = osrfHashGet( idlClass, "tablename" );
5567
5568                                         buffer_fadd( select_buf, " oils_i18n_xlate('%s', '%s', '%s', "
5569                                                         "'%s', \"%s\".%s::TEXT, '%s') AS \"%s\"",
5570                                                         tname, cname, fname, pkey, cname, pkey, locale, fname );
5571                                 } else {
5572                                         buffer_fadd( select_buf, " \"%s\".%s", cname, fname );
5573                                 }
5574                         } else {
5575                                 buffer_fadd( select_buf, " \"%s\".%s", cname, fname );
5576                         }
5577                 }
5578
5579                 jsonIteratorFree( select_itr );
5580         }
5581
5582         jsonIteratorFree( class_itr );
5583
5584         char* col_list = buffer_release( select_buf );
5585         char* table = oilsGetRelation( meta );
5586         if( !table )
5587                 table = strdup( "(null)" );
5588
5589         buffer_fadd( sql_buf, "SELECT %s FROM %s AS \"%s\"", col_list, table, core_class );
5590         free( col_list );
5591         free( table );
5592
5593         // Clear the query stack (as a fail-safe precaution against possible
5594         // leftover garbage); then push the first query frame onto the stack.
5595         clear_query_stack();
5596         push_query_frame();
5597         if( add_query_core( NULL, core_class ) ) {
5598                 if( ctx )
5599                         osrfAppSessionStatus(
5600                                 ctx->session,
5601                                 OSRF_STATUS_INTERNALSERVERERROR,
5602                                 "osrfMethodException",
5603                                 ctx->request,
5604                                 "Unable to build query frame for core class"
5605                         );
5606                 buffer_free( sql_buf );
5607                 if( defaultselhash )
5608                         jsonObjectFree( defaultselhash );
5609                 return NULL;
5610         }
5611
5612         // Add the JOIN clauses, if any
5613         if( join_hash ) {
5614                 char* join_clause = searchJOIN( join_hash, &curr_query->core );
5615                 OSRF_BUFFER_ADD_CHAR( sql_buf, ' ' );
5616                 OSRF_BUFFER_ADD( sql_buf, join_clause );
5617                 free( join_clause );
5618         }
5619
5620         osrfLogDebug( OSRF_LOG_MARK, "%s pre-predicate SQL =  %s",
5621                 modulename, OSRF_BUFFER_C_STR( sql_buf ));
5622
5623         OSRF_BUFFER_ADD( sql_buf, " WHERE " );
5624
5625         // Add the conditions in the WHERE clause
5626         char* pred = searchWHERE( search_hash, &curr_query->core, AND_OP_JOIN, ctx );
5627         if( !pred ) {
5628                 osrfAppSessionStatus(
5629                         ctx->session,
5630                         OSRF_STATUS_INTERNALSERVERERROR,
5631                                 "osrfMethodException",
5632                                 ctx->request,
5633                                 "Severe query error -- see error log for more details"
5634                         );
5635                 buffer_free( sql_buf );
5636                 if( defaultselhash )
5637                         jsonObjectFree( defaultselhash );
5638                 clear_query_stack();
5639                 return NULL;
5640         } else {
5641                 buffer_add( sql_buf, pred );
5642                 free( pred );
5643         }
5644
5645         // Add the ORDER BY, LIMIT, and/or OFFSET clauses, if present
5646         if( rest_of_query ) {
5647                 const jsonObject* order_by = NULL;
5648                 if( ( order_by = jsonObjectGetKeyConst( rest_of_query, "order_by" )) ){
5649
5650                         char* order_by_list = NULL;
5651
5652                         if( JSON_ARRAY == order_by->type ) {
5653                                 order_by_list = buildOrderByFromArray( ctx, order_by );
5654                                 if( !order_by_list ) {
5655                                         buffer_free( sql_buf );
5656                                         if( defaultselhash )
5657                                                 jsonObjectFree( defaultselhash );
5658                                         clear_query_stack();
5659                                         return NULL;
5660                                 }
5661                         } else if( JSON_HASH == order_by->type ) {
5662                                 // We expect order_by to be a JSON_HASH keyed on class names.  Traverse it
5663                                 // and build a list of ORDER BY expressions.
5664                                 growing_buffer* order_buf = buffer_init( 128 );
5665                                 first = 1;
5666                                 jsonIterator* class_itr = jsonNewIterator( order_by );
5667                                 while( (snode = jsonIteratorNext( class_itr )) ) {  // For each class:
5668
5669                                         ClassInfo* order_class_info = search_alias( class_itr->key );
5670                                         if( ! order_class_info )
5671                                                 continue;    // class not referenced by FROM clause?  Ignore it.
5672
5673                                         if( JSON_HASH == snode->type ) {
5674
5675                                                 // If the data for the current class is a JSON_HASH, then it is
5676                                                 // keyed on field name.
5677
5678                                                 const jsonObject* onode = NULL;
5679                                                 jsonIterator* order_itr = jsonNewIterator( snode );
5680                                                 while( (onode = jsonIteratorNext( order_itr )) ) {  // For each field
5681
5682                                                         osrfHash* field_def = osrfHashGet(
5683                                                                 order_class_info->fields, order_itr->key );
5684                                                         if( !field_def )
5685                                                                 continue;    // Field not defined in IDL?  Ignore it.
5686                                                         if( str_is_true( osrfHashGet( field_def, "virtual")))
5687                                                                 continue;    // Field is virtual?  Ignore it.
5688
5689                                                         char* field_str = NULL;
5690                                                         char* direction = NULL;
5691                                                         if( onode->type == JSON_HASH ) {
5692                                                                 if( jsonObjectGetKeyConst( onode, "transform" ) ) {
5693                                                                         field_str = searchFieldTransform(
5694                                                                                 class_itr->key, field_def, onode );
5695                                                                         if( ! field_str ) {
5696                                                                                 osrfAppSessionStatus(
5697                                                                                         ctx->session,
5698                                                                                         OSRF_STATUS_INTERNALSERVERERROR,
5699                                                                                         "osrfMethodException",
5700                                                                                         ctx->request,
5701                                                                                         "Severe query error in ORDER BY clause -- "
5702                                                                                         "see error log for more details"
5703                                                                                 );
5704                                                                                 jsonIteratorFree( order_itr );
5705                                                                                 jsonIteratorFree( class_itr );
5706                                                                                 buffer_free( order_buf );
5707                                                                                 buffer_free( sql_buf );
5708                                                                                 if( defaultselhash )
5709                                                                                         jsonObjectFree( defaultselhash );
5710                                                                                 clear_query_stack();
5711                                                                                 return NULL;
5712                                                                         }
5713                                                                 } else {
5714                                                                         growing_buffer* field_buf = buffer_init( 16 );
5715                                                                         buffer_fadd( field_buf, "\"%s\".%s",
5716                                                                                 class_itr->key, order_itr->key );
5717                                                                         field_str = buffer_release( field_buf );
5718                                                                 }
5719
5720                                                                 if( ( order_by = jsonObjectGetKeyConst( onode, "direction" )) ) {
5721                                                                         const char* dir = jsonObjectGetString( order_by );
5722                                                                         if(!strncasecmp( dir, "d", 1 )) {
5723                                                                                 direction = " DESC";
5724                                                                         }
5725                                                                 }
5726                                                         } else {
5727                                                                 field_str = strdup( order_itr->key );
5728                                                                 const char* dir = jsonObjectGetString( onode );
5729                                                                 if( !strncasecmp( dir, "d", 1 )) {
5730                                                                         direction = " DESC";
5731                                                                 } else {
5732                                                                         direction = " ASC";
5733                                                                 }
5734                                                         }
5735
5736                                                         if( first ) {
5737                                                                 first = 0;
5738                                                         } else {
5739                                                                 buffer_add( order_buf, ", " );
5740                                                         }
5741
5742                                                         buffer_add( order_buf, field_str );
5743                                                         free( field_str );
5744
5745                                                         if( direction ) {
5746                                                                 buffer_add( order_buf, direction );
5747                                                         }
5748                                                 } // end while; looping over ORDER BY expressions
5749
5750                                                 jsonIteratorFree( order_itr );
5751
5752                                         } else if( JSON_STRING == snode->type ) {
5753                                                 // We expect a comma-separated list of sort fields.
5754                                                 const char* str = jsonObjectGetString( snode );
5755                                                 if( strchr( str, ';' )) {
5756                                                         // No semicolons allowed.  It is theoretically possible for a
5757                                                         // legitimate semicolon to occur within quotes, but it's not likely
5758                                                         // to occur in practice in the context of an ORDER BY list.
5759                                                         osrfLogError( OSRF_LOG_MARK, "%s: Possible attempt at SOL injection -- "
5760                                                                 "semicolon found in ORDER BY list: \"%s\"", modulename, str );
5761                                                         if( ctx ) {
5762                                                                 osrfAppSessionStatus(
5763                                                                         ctx->session,
5764                                                                         OSRF_STATUS_INTERNALSERVERERROR,
5765                                                                         "osrfMethodException",
5766                                                                         ctx->request,
5767                                                                         "Possible attempt at SOL injection -- "
5768                                                                                 "semicolon found in ORDER BY list"
5769                                                                 );
5770                                                         }
5771                                                         jsonIteratorFree( class_itr );
5772                                                         buffer_free( order_buf );
5773                                                         buffer_free( sql_buf );
5774                                                         if( defaultselhash )
5775                                                                 jsonObjectFree( defaultselhash );
5776                                                         clear_query_stack();
5777                                                         return NULL;
5778                                                 }
5779                                                 buffer_add( order_buf, str );
5780                                                 break;
5781                                         }
5782
5783                                 } // end while; looping over order_by classes
5784
5785                                 jsonIteratorFree( class_itr );
5786                                 order_by_list = buffer_release( order_buf );
5787
5788                         } else {
5789                                 osrfLogWarning( OSRF_LOG_MARK,
5790                                         "\"order_by\" object in a query is not a JSON_HASH or JSON_ARRAY;"
5791                                         "no ORDER BY generated" );
5792                         }
5793
5794                         if( order_by_list && *order_by_list ) {
5795                                 OSRF_BUFFER_ADD( sql_buf, " ORDER BY " );
5796                                 OSRF_BUFFER_ADD( sql_buf, order_by_list );
5797                         }
5798
5799                         free( order_by_list );
5800                 }
5801
5802                 const jsonObject* limit = jsonObjectGetKeyConst( rest_of_query, "limit" );
5803                 if( limit ) {
5804                         const char* str = jsonObjectGetString( limit );
5805                         if (str) {
5806                                 buffer_fadd(
5807                                         sql_buf,
5808                                         " LIMIT %d",
5809                                         atoi(str)
5810                                 );
5811                         }
5812                 }
5813
5814                 const jsonObject* offset = jsonObjectGetKeyConst( rest_of_query, "offset" );
5815                 if( offset ) {
5816                         const char* str = jsonObjectGetString( offset );
5817                         if (str) {
5818                                 buffer_fadd(
5819                                         sql_buf,
5820                                         " OFFSET %d",
5821                                         atoi( str )
5822                                 );
5823                         }
5824                 }
5825         }
5826
5827         if( defaultselhash )
5828                 jsonObjectFree( defaultselhash );
5829         clear_query_stack();
5830
5831         OSRF_BUFFER_ADD_CHAR( sql_buf, ';' );
5832         return buffer_release( sql_buf );
5833 }
5834
5835 int doJSONSearch ( osrfMethodContext* ctx ) {
5836         if(osrfMethodVerifyContext( ctx )) {
5837                 osrfLogError( OSRF_LOG_MARK,  "Invalid method context" );
5838                 return -1;
5839         }
5840
5841         osrfLogDebug( OSRF_LOG_MARK, "Received query request" );
5842
5843         int err = 0;
5844
5845         jsonObject* hash = jsonObjectGetIndex( ctx->params, 0 );
5846
5847         int flags = 0;
5848
5849         if( obj_is_true( jsonObjectGetKeyConst( hash, "distinct" )))
5850                 flags |= SELECT_DISTINCT;
5851
5852         if( obj_is_true( jsonObjectGetKeyConst( hash, "no_i18n" )))
5853                 flags |= DISABLE_I18N;
5854
5855         osrfLogDebug( OSRF_LOG_MARK, "Building SQL ..." );
5856         clear_query_stack();       // a possibly needless precaution
5857         char* sql = buildQuery( ctx, hash, flags );
5858         clear_query_stack();
5859
5860         if( !sql ) {
5861                 err = -1;
5862                 return err;
5863         }
5864
5865         osrfLogDebug( OSRF_LOG_MARK, "%s SQL =  %s", modulename, sql );
5866
5867         // XXX for now...
5868         dbhandle = writehandle;
5869
5870         dbi_result result = dbi_conn_query( dbhandle, sql );
5871
5872         if( result ) {
5873                 osrfLogDebug( OSRF_LOG_MARK, "Query returned with no errors" );
5874
5875                 if( dbi_result_first_row( result )) {
5876                         /* JSONify the result */
5877                         osrfLogDebug( OSRF_LOG_MARK, "Query returned at least one row" );
5878
5879                         do {
5880                                 jsonObject* return_val = oilsMakeJSONFromResult( result );
5881                                 osrfAppRespond( ctx, return_val );
5882                                 jsonObjectFree( return_val );
5883                         } while( dbi_result_next_row( result ));
5884
5885                 } else {
5886                         osrfLogDebug( OSRF_LOG_MARK, "%s returned no results for query %s", modulename, sql );
5887                 }
5888
5889                 osrfAppRespondComplete( ctx, NULL );
5890
5891                 /* clean up the query */
5892                 dbi_result_free( result );
5893
5894         } else {
5895                 err = -1;
5896                 const char* msg;
5897                 int errnum = dbi_conn_error( dbhandle, &msg );
5898                 osrfLogError( OSRF_LOG_MARK, "%s: Error with query [%s]: %d %s",
5899                         modulename, sql, errnum, msg ? msg : "(No description available)" );
5900                 osrfAppSessionStatus(
5901                         ctx->session,
5902                         OSRF_STATUS_INTERNALSERVERERROR,
5903                         "osrfMethodException",
5904                         ctx->request,
5905                         "Severe query error -- see error log for more details"
5906                 );
5907                 if( !oilsIsDBConnected( dbhandle ))
5908                         osrfAppSessionPanic( ctx->session );
5909         }
5910
5911         free( sql );
5912         return err;
5913 }
5914
5915 // The last parameter, err, is used to report an error condition by updating an int owned by
5916 // the calling code.
5917
5918 // In case of an error, we set *err to -1.  If there is no error, *err is left unchanged.
5919 // It is the responsibility of the calling code to initialize *err before the
5920 // call, so that it will be able to make sense of the result.
5921
5922 // Note also that we return NULL if and only if we set *err to -1.  So the err parameter is
5923 // redundant anyway.
5924 static jsonObject* doFieldmapperSearch( osrfMethodContext* ctx, osrfHash* class_meta,
5925                 jsonObject* where_hash, jsonObject* query_hash, int* err ) {
5926
5927         const char* tz = _sanitize_tz_name(ctx->session->session_tz);
5928
5929         // XXX for now...
5930         dbhandle = writehandle;
5931
5932         char* core_class = osrfHashGet( class_meta, "classname" );
5933         osrfLogDebug( OSRF_LOG_MARK, "entering doFieldmapperSearch() with core_class %s", core_class );
5934
5935         char* pkey = osrfHashGet( class_meta, "primarykey" );
5936
5937         if (!ctx->session->userData)
5938                 (void) initSessionCache( ctx );
5939
5940         char *methodtype = osrfHashGet( (osrfHash *) ctx->method->userData, "methodtype" );
5941         char *inside_verify = osrfHashGet( (osrfHash*) ctx->session->userData, "inside_verify" );
5942         int need_to_verify = (inside_verify ? !atoi(inside_verify) : 1);
5943
5944         int i_respond_directly = 0;
5945         int flesh_depth = 0;
5946
5947         char* sql = buildSELECT( where_hash, query_hash, class_meta, ctx );
5948         if( !sql ) {
5949                 osrfLogDebug( OSRF_LOG_MARK, "Problem building query, returning NULL" );
5950                 *err = -1;
5951                 return NULL;
5952         }
5953
5954         osrfLogDebug( OSRF_LOG_MARK, "%s SQL =  %s", modulename, sql );
5955
5956         // Setting the timezone if requested and not in a transaction
5957         if (!getXactId(ctx)) {
5958                 if (tz) {
5959                         setenv("TZ",tz,1);
5960                         tzset();
5961                         dbi_result tz_res = dbi_conn_queryf( writehandle, "SET timezone TO '%s'; -- cstore", tz );
5962                         if( !tz_res ) {
5963                                 osrfLogError( OSRF_LOG_MARK, "%s: Error setting timezone %s", modulename, tz);
5964                                 osrfAppSessionStatus( ctx->session, OSRF_STATUS_INTERNALSERVERERROR,
5965                                         "osrfMethodException", ctx->request, "Error setting timezone" );
5966                                 if( !oilsIsDBConnected( writehandle )) {
5967                                         osrfAppSessionPanic( ctx->session );
5968                                         return NULL;
5969                                 }
5970                         } else {
5971                                 dbi_result_free( tz_res );
5972                         }
5973                 } else {
5974                         unsetenv("TZ");
5975                         tzset();
5976                         dbi_result res = dbi_conn_queryf( writehandle, "SET timezone TO DEFAULT; -- cstore" );
5977                         if( !res ) {
5978                                 osrfLogError( OSRF_LOG_MARK, "%s: Error resetting timezone", modulename);
5979                                 if( !oilsIsDBConnected( writehandle )) {
5980                                         osrfAppSessionPanic( ctx->session );
5981                                         return NULL;
5982                                 }
5983                         } else {
5984                                 dbi_result_free( res );
5985                         }
5986                 }
5987         }
5988
5989
5990         dbi_result result = dbi_conn_query( dbhandle, sql );
5991
5992         if( NULL == result ) {
5993                 const char* msg;
5994                 int errnum = dbi_conn_error( dbhandle, &msg );
5995                 osrfLogError(OSRF_LOG_MARK, "%s: Error retrieving %s with query [%s]: %d %s",
5996                         modulename, osrfHashGet( class_meta, "fieldmapper" ), sql, errnum,
5997                         msg ? msg : "(No description available)" );
5998                 if( !oilsIsDBConnected( dbhandle ))
5999                         osrfAppSessionPanic( ctx->session );
6000                 osrfAppSessionStatus(
6001                         ctx->session,
6002                         OSRF_STATUS_INTERNALSERVERERROR,
6003                         "osrfMethodException",
6004                         ctx->request,
6005                         "Severe query error -- see error log for more details"
6006                 );
6007                 *err = -1;
6008                 free( sql );
6009                 return NULL;
6010
6011         } else {
6012                 osrfLogDebug( OSRF_LOG_MARK, "Query returned with no errors" );
6013
6014         }
6015
6016         jsonObject* res_list = jsonNewObjectType( JSON_ARRAY );
6017         jsonObject* row_obj = NULL;
6018
6019         // The following two steps are for verifyObjectPCRUD()'s benefit.
6020         // 1. get the flesh depth
6021         const jsonObject* _tmp = jsonObjectGetKeyConst( query_hash, "flesh" );
6022         if( _tmp ) {
6023                 flesh_depth = (int) jsonObjectGetNumber( _tmp );
6024                 if( flesh_depth == -1 || flesh_depth > max_flesh_depth )
6025                         flesh_depth = max_flesh_depth;
6026         }
6027
6028         // 2. figure out one consistent rs_size for verifyObjectPCRUD to use
6029         // over the whole life of this request.  This means if we've already set
6030         // up a rs_size_req_%d, do nothing.
6031         //      a. Incidentally, we can also use this opportunity to set i_respond_directly
6032         int *rs_size = osrfHashGetFmt( (osrfHash *) ctx->session->userData, "rs_size_req_%d", ctx->request );
6033         if( !rs_size ) {        // pointer null, so value not set in hash
6034                 // i_respond_directly can only be true at the /top/ of a recursive search, if even that.
6035                 i_respond_directly = ( *methodtype == 'r' || *methodtype == 'i' || *methodtype == 's' );
6036
6037                 rs_size = (int *) safe_malloc( sizeof(int) );   // will be freed by sessionDataFree()
6038                 unsigned long long result_count = dbi_result_get_numrows( result );
6039                 *rs_size = (int) result_count * (flesh_depth + 1);      // yes, we could lose some bits, but come on
6040                 osrfHashSet( (osrfHash *) ctx->session->userData, rs_size, "rs_size_req_%d", ctx->request );
6041         }
6042
6043         if( dbi_result_first_row( result )) {
6044
6045                 // Convert each row to a JSON_ARRAY of column values, and enclose those objects
6046                 // in a JSON_ARRAY of rows.  If two or more rows have the same key value, then
6047                 // eliminate the duplicates.
6048                 osrfLogDebug( OSRF_LOG_MARK, "Query returned at least one row" );
6049                 osrfHash* dedup = osrfNewHash();
6050                 do {
6051                         row_obj = oilsMakeFieldmapperFromResult( result, class_meta );
6052                         char* pkey_val = oilsFMGetString( row_obj, pkey );
6053                         if( osrfHashGet( dedup, pkey_val ) ) {
6054                                 jsonObjectFree( row_obj );
6055                                 free( pkey_val );
6056                         } else {
6057                                 if( !enforce_pcrud || !need_to_verify ||
6058                                                 verifyObjectPCRUD( ctx, class_meta, row_obj, 0 /* means check user data for rs_size */ )) {
6059                                         osrfHashSet( dedup, pkey_val, pkey_val );
6060                                         jsonObjectPush( res_list, row_obj );
6061                                 }
6062                         }
6063                 } while( dbi_result_next_row( result ));
6064                 osrfHashFree( dedup );
6065
6066         } else {
6067                 osrfLogDebug( OSRF_LOG_MARK, "%s returned no results for query %s",
6068                         modulename, sql );
6069         }
6070
6071         /* clean up the query */
6072         dbi_result_free( result );
6073         free( sql );
6074
6075         // If we're asked to flesh, and there's anything to flesh, then flesh it
6076         // (formerly we would skip fleshing if in pcrud mode, but now we support
6077         // fleshing even in PCRUD).
6078         if( res_list->size ) {
6079                 jsonObject* temp_blob;  // We need a non-zero flesh depth, and a list of fields to flesh
6080                 jsonObject* flesh_fields;
6081                 jsonObject* flesh_blob = NULL;
6082                 osrfStringArray* link_fields = NULL;
6083                 osrfHash* links = NULL;
6084                 int want_flesh = 0;
6085
6086                 if( query_hash ) {
6087                         temp_blob = jsonObjectGetKey( query_hash, "flesh_fields" );
6088                         if( temp_blob && flesh_depth > 0 ) {
6089
6090                                 flesh_blob = jsonObjectClone( temp_blob );
6091                                 flesh_fields = jsonObjectGetKey( flesh_blob, core_class );
6092
6093                                 links = osrfHashGet( class_meta, "links" );
6094
6095                                 // Make an osrfStringArray of the names of fields to be fleshed
6096                                 if( flesh_fields ) {
6097                                         if( flesh_fields->size == 1 ) {
6098                                                 const char* _t = jsonObjectGetString(
6099                                                         jsonObjectGetIndex( flesh_fields, 0 ) );
6100                                                 if( !strcmp( _t, "*" ))
6101                                                         link_fields = osrfHashKeys( links );
6102                                         }
6103
6104                                         if( !link_fields ) {
6105                                                 jsonObject* _f;
6106                                                 link_fields = osrfNewStringArray( 1 );
6107                                                 jsonIterator* _i = jsonNewIterator( flesh_fields );
6108                                                 while ((_f = jsonIteratorNext( _i ))) {
6109                                                         osrfStringArrayAdd( link_fields, jsonObjectGetString( _f ) );
6110                                                 }
6111                                                 jsonIteratorFree( _i );
6112                                         }
6113                                 }
6114                                 want_flesh = link_fields ? 1 : 0;
6115                         }
6116                 }
6117
6118                 osrfHash* fields = osrfHashGet( class_meta, "fields" );
6119
6120                 // Iterate over the JSON_ARRAY of rows
6121                 jsonObject* cur;
6122                 unsigned long res_idx = 0;
6123                 while((cur = jsonObjectGetIndex( res_list, res_idx++ ) )) {
6124
6125                         int i = 0;
6126                         const char* link_field;
6127
6128                         // Iterate over the list of fleshable fields
6129                         if ( want_flesh ) {
6130                                 while( (link_field = osrfStringArrayGetString(link_fields, i++)) ) {
6131
6132                                         osrfLogDebug( OSRF_LOG_MARK, "Starting to flesh %s", link_field );
6133
6134                                         osrfHash* kid_link = osrfHashGet( links, link_field );
6135                                         if( !kid_link )
6136                                                 continue;     // Not a link field; skip it
6137
6138                                         osrfHash* field = osrfHashGet( fields, link_field );
6139                                         if( !field )
6140                                                 continue;     // Not a field at all; skip it (IDL is ill-formed)
6141
6142                                         osrfHash* kid_idl = osrfHashGet( oilsIDL(),
6143                                                 osrfHashGet( kid_link, "class" ));
6144                                         if( !kid_idl )
6145                                                 continue;   // The class it links to doesn't exist; skip it
6146
6147                                         const char* reltype = osrfHashGet( kid_link, "reltype" );
6148                                         if( !reltype )
6149                                                 continue;   // No reltype; skip it (IDL is ill-formed)
6150
6151                                         osrfHash* value_field = field;
6152
6153                                         if(    !strcmp( reltype, "has_many" )
6154                                                 || !strcmp( reltype, "might_have" ) ) { // has_many or might_have
6155                                                 value_field = osrfHashGet(
6156                                                         fields, osrfHashGet( class_meta, "primarykey" ) );
6157                                         }
6158
6159                                         int kid_has_controller = osrfStringArrayContains( osrfHashGet(kid_idl, "controller"), modulename );
6160                                         // fleshing pcrud case: we require the controller in need_to_verify mode
6161                                         if ( !kid_has_controller && enforce_pcrud && need_to_verify ) {
6162                                                 osrfLogInfo( OSRF_LOG_MARK, "%s is not listed as a controller for %s; moving on", modulename, core_class );
6163
6164                                                 jsonObjectSetIndex(
6165                                                         cur,
6166                                                         (unsigned long) atoi( osrfHashGet(field, "array_position") ),
6167                                                         jsonNewObjectType(
6168                                                                 !strcmp( reltype, "has_many" ) ? JSON_ARRAY : JSON_NULL
6169                                                         )
6170                                                 );
6171                                                 continue;
6172                                         }
6173
6174                                         osrfStringArray* link_map = osrfHashGet( kid_link, "map" );
6175
6176                                         if( link_map->size > 0 ) {
6177                                                 jsonObject* _kid_key = jsonNewObjectType( JSON_ARRAY );
6178                                                 jsonObjectPush(
6179                                                         _kid_key,
6180                                                         jsonNewObject( osrfStringArrayGetString( link_map, 0 ) )
6181                                                 );
6182
6183                                                 jsonObjectSetKey(
6184                                                         flesh_blob,
6185                                                         osrfHashGet( kid_link, "class" ),
6186                                                         _kid_key
6187                                                 );
6188                                         };
6189
6190                                         osrfLogDebug(
6191                                                 OSRF_LOG_MARK,
6192                                                 "Link field: %s, remote class: %s, fkey: %s, reltype: %s",
6193                                                 osrfHashGet( kid_link, "field" ),
6194                                                 osrfHashGet( kid_link, "class" ),
6195                                                 osrfHashGet( kid_link, "key" ),
6196                                                 osrfHashGet( kid_link, "reltype" )
6197                                         );
6198
6199                                         const char* search_key = jsonObjectGetString(
6200                                                 jsonObjectGetIndex( cur,
6201                                                         atoi( osrfHashGet( value_field, "array_position" ) )
6202                                                 )
6203                                         );
6204
6205                                         if( !search_key ) {
6206                                                 osrfLogDebug( OSRF_LOG_MARK, "Nothing to search for!" );
6207                                                 continue;
6208                                         }
6209
6210                                         osrfLogDebug( OSRF_LOG_MARK, "Creating param objects..." );
6211
6212                                         // construct WHERE clause
6213                                         jsonObject* where_clause  = jsonNewObjectType( JSON_HASH );
6214                                         jsonObjectSetKey(
6215                                                 where_clause,
6216                                                 osrfHashGet( kid_link, "key" ),
6217                                                 jsonNewObject( search_key )
6218                                         );
6219
6220                                         // construct the rest of the query, mostly
6221                                         // by copying pieces of the previous level of query
6222                                         jsonObject* rest_of_query = jsonNewObjectType( JSON_HASH );
6223                                         jsonObjectSetKey( rest_of_query, "flesh",
6224                                                 jsonNewNumberObject( flesh_depth - 1 + link_map->size )
6225                                         );
6226
6227                                         if( flesh_blob )
6228                                                 jsonObjectSetKey( rest_of_query, "flesh_fields",
6229                                                         jsonObjectClone( flesh_blob ));
6230
6231                                         if( jsonObjectGetKeyConst( query_hash, "order_by" )) {
6232                                                 jsonObjectSetKey( rest_of_query, "order_by",
6233                                                         jsonObjectClone( jsonObjectGetKeyConst( query_hash, "order_by" ))
6234                                                 );
6235                                         }
6236
6237                                         if( jsonObjectGetKeyConst( query_hash, "select" )) {
6238                                                 jsonObjectSetKey( rest_of_query, "select",
6239                                                         jsonObjectClone( jsonObjectGetKeyConst( query_hash, "select" ))
6240                                                 );
6241                                         }
6242
6243                                         // do the query, recursively, to expand the fleshable field
6244                                         jsonObject* kids = doFieldmapperSearch( ctx, kid_idl,
6245                                                 where_clause, rest_of_query, err );
6246
6247                                         jsonObjectFree( where_clause );
6248                                         jsonObjectFree( rest_of_query );
6249
6250                                         if( *err ) {
6251                                                 osrfStringArrayFree( link_fields );
6252                                                 jsonObjectFree( res_list );
6253                                                 jsonObjectFree( flesh_blob );
6254                                                 return NULL;
6255                                         }
6256
6257                                         osrfLogDebug( OSRF_LOG_MARK, "Search for %s return %d linked objects",
6258                                                 osrfHashGet( kid_link, "class" ), kids->size );
6259
6260                                         // Traverse the result set
6261                                         jsonObject* X = NULL;
6262                                         if( link_map->size > 0 && kids->size > 0 ) {
6263                                                 X = kids;
6264                                                 kids = jsonNewObjectType( JSON_ARRAY );
6265
6266                                                 jsonObject* _k_node;
6267                                                 unsigned long res_idx = 0;
6268                                                 while((_k_node = jsonObjectGetIndex( X, res_idx++ ) )) {
6269                                                         jsonObjectPush(
6270                                                                 kids,
6271                                                                 jsonObjectClone(
6272                                                                         jsonObjectGetIndex(
6273                                                                                 _k_node,
6274                                                                                 (unsigned long) atoi(
6275                                                                                         osrfHashGet(
6276                                                                                                 osrfHashGet(
6277                                                                                                         osrfHashGet(
6278                                                                                                                 osrfHashGet(
6279                                                                                                                         oilsIDL(),
6280                                                                                                                         osrfHashGet( kid_link, "class" )
6281                                                                                                                 ),
6282                                                                                                                 "fields"
6283                                                                                                         ),
6284                                                                                                         osrfStringArrayGetString( link_map, 0 )
6285                                                                                                 ),
6286                                                                                                 "array_position"
6287                                                                                         )
6288                                                                                 )
6289                                                                         )
6290                                                                 )
6291                                                         );
6292                                                 } // end while loop traversing X
6293                                         }
6294
6295                                         if (kids->size > 0) {
6296
6297                                                 if((   !strcmp( osrfHashGet( kid_link, "reltype" ), "has_a" )
6298                                                         || !strcmp( osrfHashGet( kid_link, "reltype" ), "might_have" ))
6299                                                 ) {
6300                                                         osrfLogDebug(OSRF_LOG_MARK, "Storing fleshed objects in %s",
6301                                                                 osrfHashGet( kid_link, "field" ));
6302                                                         jsonObjectSetIndex(
6303                                                                 cur,
6304                                                                 (unsigned long) atoi( osrfHashGet( field, "array_position" ) ),
6305                                                                 jsonObjectClone( jsonObjectGetIndex( kids, 0 ))
6306                                                         );
6307                                                 }
6308                                         }
6309
6310                                         if( !strcmp( osrfHashGet( kid_link, "reltype" ), "has_many" )) {
6311                                                 // has_many
6312                                                 osrfLogDebug( OSRF_LOG_MARK, "Storing fleshed objects in %s",
6313                                                         osrfHashGet( kid_link, "field" ) );
6314                                                 jsonObjectSetIndex(
6315                                                         cur,
6316                                                         (unsigned long) atoi( osrfHashGet( field, "array_position" ) ),
6317                                                         jsonObjectClone( kids )
6318                                                 );
6319                                         }
6320
6321                                         if( X ) {
6322                                                 jsonObjectFree( kids );
6323                                                 kids = X;
6324                                         }
6325
6326                                         jsonObjectFree( kids );
6327
6328                                         osrfLogDebug( OSRF_LOG_MARK, "Fleshing of %s complete",
6329                                                 osrfHashGet( kid_link, "field" ) );
6330                                         osrfLogDebug( OSRF_LOG_MARK, "%s", jsonObjectToJSON( cur ));
6331
6332                                 } // end while loop traversing list of fleshable fields
6333                         }
6334
6335                         if( i_respond_directly ) {
6336                                 if ( *methodtype == 'i' ) {
6337                                         osrfAppRespond( ctx,
6338                                                 oilsFMGetObject( cur, osrfHashGet( class_meta, "primarykey" ) ) );
6339                                 } else {
6340                                         osrfAppRespond( ctx, cur );
6341                                 }
6342                         }
6343                 } // end while loop traversing res_list
6344                 jsonObjectFree( flesh_blob );
6345                 osrfStringArrayFree( link_fields );
6346         }
6347
6348         if( i_respond_directly ) {
6349                 jsonObjectFree( res_list );
6350                 return jsonNewObjectType( JSON_ARRAY );
6351         } else {
6352                 return res_list;
6353         }
6354 }
6355
6356
6357 int doUpdate( osrfMethodContext* ctx ) {
6358         if( osrfMethodVerifyContext( ctx )) {
6359                 osrfLogError( OSRF_LOG_MARK, "Invalid method context" );
6360                 return -1;
6361         }
6362
6363         if( enforce_pcrud )
6364                 timeout_needs_resetting = 1;
6365
6366         osrfHash* meta = osrfHashGet( (osrfHash*) ctx->method->userData, "class" );
6367
6368         jsonObject* target = NULL;
6369         if( enforce_pcrud )
6370                 target = jsonObjectGetIndex( ctx->params, 1 );
6371         else
6372                 target = jsonObjectGetIndex( ctx->params, 0 );
6373
6374         if(!verifyObjectClass( ctx, target )) {
6375                 osrfAppRespondComplete( ctx, NULL );
6376                 return -1;
6377         }
6378
6379         if( getXactId( ctx ) == NULL ) {
6380                 osrfAppSessionStatus(
6381                         ctx->session,
6382                         OSRF_STATUS_BADREQUEST,
6383                         "osrfMethodException",
6384                         ctx->request,
6385                         "No active transaction -- required for UPDATE"
6386                 );
6387                 osrfAppRespondComplete( ctx, NULL );
6388                 return -1;
6389         }
6390
6391         // The following test is harmless but redundant.  If a class is
6392         // readonly, we don't register an update method for it.
6393         if( str_is_true( osrfHashGet( meta, "readonly" ) ) ) {
6394                 osrfAppSessionStatus(
6395                         ctx->session,
6396                         OSRF_STATUS_BADREQUEST,
6397                         "osrfMethodException",
6398                         ctx->request,
6399                         "Cannot UPDATE readonly class"
6400                 );
6401                 osrfAppRespondComplete( ctx, NULL );
6402                 return -1;
6403         }
6404
6405         const char* trans_id = getXactId( ctx );
6406
6407         // Set the last_xact_id
6408         int index = oilsIDL_ntop( target->classname, "last_xact_id" );
6409         if( index > -1 ) {
6410                 osrfLogDebug( OSRF_LOG_MARK, "Setting last_xact_id to %s on %s at position %d",
6411                                 trans_id, target->classname, index );
6412                 jsonObjectSetIndex( target, index, jsonNewObject( trans_id ));
6413         }
6414
6415         char* pkey = osrfHashGet( meta, "primarykey" );
6416         osrfHash* fields = osrfHashGet( meta, "fields" );
6417
6418         char* id = oilsFMGetString( target, pkey );
6419
6420         osrfLogDebug(
6421                 OSRF_LOG_MARK,
6422                 "%s updating %s object with %s = %s",
6423                 modulename,
6424                 osrfHashGet( meta, "fieldmapper" ),
6425                 pkey,
6426                 id
6427         );
6428
6429         dbhandle = writehandle;
6430         growing_buffer* sql = buffer_init( 128 );
6431         buffer_fadd( sql,"UPDATE %s SET", osrfHashGet( meta, "tablename" ));
6432
6433         int first = 1;
6434         osrfHash* field_def = NULL;
6435         osrfHashIterator* field_itr = osrfNewHashIterator( fields );
6436         while( ( field_def = osrfHashIteratorNext( field_itr ) ) ) {
6437
6438                 // Skip virtual fields, and the primary key
6439                 if( str_is_true( osrfHashGet( field_def, "virtual") ) )
6440                         continue;
6441
6442                 if (osrfStringArrayContains( osrfHashGet(field_def, "suppress_controller"), modulename ))
6443                         continue;
6444
6445
6446                 const char* field_name = osrfHashIteratorKey( field_itr );
6447                 if( ! strcmp( field_name, pkey ) )
6448                         continue;
6449
6450                 const jsonObject* field_object = oilsFMGetObject( target, field_name );
6451
6452                 int value_is_numeric = 0;    // boolean
6453                 char* value;
6454                 if( field_object && field_object->classname ) {
6455                         value = oilsFMGetString(
6456                                 field_object,
6457                                 (char*) oilsIDLFindPath( "/%s/primarykey", field_object->classname )
6458                         );
6459                 } else if( field_object && JSON_BOOL == field_object->type ) {
6460                         if( jsonBoolIsTrue( field_object ) )
6461                                 value = strdup( "t" );
6462                         else
6463                                 value = strdup( "f" );
6464                 } else {
6465                         value = jsonObjectToSimpleString( field_object );
6466                         if( field_object && JSON_NUMBER == field_object->type )
6467                                 value_is_numeric = 1;
6468                 }
6469
6470                 osrfLogDebug( OSRF_LOG_MARK, "Updating %s object with %s = %s",
6471                                 osrfHashGet( meta, "fieldmapper" ), field_name, value);
6472
6473                 if( !field_object || field_object->type == JSON_NULL ) {
6474                         if( !( !( strcmp( osrfHashGet( meta, "classname" ), "au" ) )
6475                                         && !( strcmp( field_name, "passwd" ) )) ) { // arg at the special case!
6476                                 if( first )
6477                                         first = 0;
6478                                 else
6479                                         OSRF_BUFFER_ADD_CHAR( sql, ',' );
6480                                 buffer_fadd( sql, " %s = NULL", field_name );
6481                         }
6482
6483                 } else if( value_is_numeric || !strcmp( get_primitive( field_def ), "number") ) {
6484                         if( first )
6485                                 first = 0;
6486                         else
6487                                 OSRF_BUFFER_ADD_CHAR( sql, ',' );
6488
6489                         const char* numtype = get_datatype( field_def );
6490                         if( !strncmp( numtype, "INT", 3 ) ) {
6491                                 buffer_fadd( sql, " %s = %ld", field_name, atol( value ) );
6492                         } else if( !strcmp( numtype, "NUMERIC" ) ) {
6493                                 buffer_fadd( sql, " %s = %f", field_name, atof( value ) );
6494                         } else {
6495                                 // Must really be intended as a string, so quote it
6496                                 if( dbi_conn_quote_string( dbhandle, &value )) {
6497                                         buffer_fadd( sql, " %s = %s", field_name, value );
6498                                 } else {
6499                                         osrfLogError( OSRF_LOG_MARK, "%s: Error quoting string [%s]",
6500                                                 modulename, value );
6501                                         osrfAppSessionStatus(
6502                                                 ctx->session,
6503                                                 OSRF_STATUS_INTERNALSERVERERROR,
6504                                                 "osrfMethodException",
6505                                                 ctx->request,
6506                                                 "Error quoting string -- please see the error log for more details"
6507                                         );
6508                                         free( value );
6509                                         free( id );
6510                                         osrfHashIteratorFree( field_itr );
6511                                         buffer_free( sql );
6512                                         osrfAppRespondComplete( ctx, NULL );
6513                                         return -1;
6514                                 }
6515                         }
6516
6517                         osrfLogDebug( OSRF_LOG_MARK, "%s is of type %s", field_name, numtype );
6518
6519                 } else {
6520                         if( dbi_conn_quote_string( dbhandle, &value ) ) {
6521                                 if( first )
6522                                         first = 0;
6523                                 else
6524                                         OSRF_BUFFER_ADD_CHAR( sql, ',' );
6525                                 buffer_fadd( sql, " %s = %s", field_name, value );
6526                         } else {
6527                                 osrfLogError( OSRF_LOG_MARK, "%s: Error quoting string [%s]", modulename, value );
6528                                 osrfAppSessionStatus(
6529                                         ctx->session,
6530                                         OSRF_STATUS_INTERNALSERVERERROR,
6531                                         "osrfMethodException",
6532                                         ctx->request,
6533                                         "Error quoting string -- please see the error log for more details"
6534                                 );
6535                                 free( value );
6536                                 free( id );
6537                                 osrfHashIteratorFree( field_itr );
6538                                 buffer_free( sql );
6539                                 osrfAppRespondComplete( ctx, NULL );
6540                                 return -1;
6541                         }
6542                 }
6543
6544                 free( value );
6545
6546         } // end while
6547
6548         osrfHashIteratorFree( field_itr );
6549
6550         jsonObject* obj = jsonNewObject( id );
6551
6552         if( strcmp( get_primitive( osrfHashGet( osrfHashGet(meta, "fields"), pkey )), "number" ))
6553                 dbi_conn_quote_string( dbhandle, &id );
6554
6555         buffer_fadd( sql, " WHERE %s = %s;", pkey, id );
6556
6557         char* query = buffer_release( sql );
6558         osrfLogDebug( OSRF_LOG_MARK, "%s: Update SQL [%s]", modulename, query );
6559
6560         dbi_result result = dbi_conn_query( dbhandle, query );
6561         free( query );
6562
6563         int rc = 0;
6564         if( !result ) {
6565                 jsonObjectFree( obj );
6566                 obj = jsonNewObject( NULL );
6567                 const char* msg;
6568                 int errnum = dbi_conn_error( dbhandle, &msg );
6569                 osrfLogError(
6570                         OSRF_LOG_MARK,
6571                         "%s ERROR updating %s object with %s = %s: %d %s",
6572                         modulename,
6573                         osrfHashGet( meta, "fieldmapper" ),
6574                         pkey,
6575                         id,
6576                         errnum,
6577                         msg ? msg : "(No description available)"
6578                 );
6579                 osrfAppSessionStatus(
6580                         ctx->session,
6581                         OSRF_STATUS_INTERNALSERVERERROR,
6582                         "osrfMethodException",
6583                         ctx->request,
6584                         "Error in updating a row -- please see the error log for more details"
6585                 );
6586                 if( !oilsIsDBConnected( dbhandle ))
6587                         osrfAppSessionPanic( ctx->session );
6588                 rc = -1;
6589         } else
6590                 dbi_result_free( result );
6591
6592         free( id );
6593         osrfAppRespondComplete( ctx, obj );
6594         jsonObjectFree( obj );
6595         return rc;
6596 }
6597
6598 int doDelete( osrfMethodContext* ctx ) {
6599         if( osrfMethodVerifyContext( ctx )) {
6600                 osrfLogError( OSRF_LOG_MARK, "Invalid method context" );
6601                 return -1;
6602         }
6603
6604         if( enforce_pcrud )
6605                 timeout_needs_resetting = 1;
6606
6607         osrfHash* meta = osrfHashGet( (osrfHash*) ctx->method->userData, "class" );
6608
6609         if( getXactId( ctx ) == NULL ) {
6610                 osrfAppSessionStatus(
6611                         ctx->session,
6612                         OSRF_STATUS_BADREQUEST,
6613                         "osrfMethodException",
6614                         ctx->request,
6615                         "No active transaction -- required for DELETE"
6616                 );
6617                 osrfAppRespondComplete( ctx, NULL );
6618                 return -1;
6619         }
6620
6621         // The following test is harmless but redundant.  If a class is
6622         // readonly, we don't register a delete method for it.
6623         if( str_is_true( osrfHashGet( meta, "readonly" ) ) ) {
6624                 osrfAppSessionStatus(
6625                         ctx->session,
6626                         OSRF_STATUS_BADREQUEST,
6627                         "osrfMethodException",
6628                         ctx->request,
6629                         "Cannot DELETE readonly class"
6630                 );
6631                 osrfAppRespondComplete( ctx, NULL );
6632                 return -1;
6633         }
6634
6635         dbhandle = writehandle;
6636
6637         char* pkey = osrfHashGet( meta, "primarykey" );
6638
6639         int _obj_pos = 0;
6640         if( enforce_pcrud )
6641                 _obj_pos = 1;
6642
6643         char* id;
6644         if( jsonObjectGetIndex( ctx->params, _obj_pos )->classname ) {
6645                 if( !verifyObjectClass( ctx, jsonObjectGetIndex( ctx->params, _obj_pos ))) {
6646                         osrfAppRespondComplete( ctx, NULL );
6647                         return -1;
6648                 }
6649
6650                 id = oilsFMGetString( jsonObjectGetIndex(ctx->params, _obj_pos), pkey );
6651         } else {
6652                 if( enforce_pcrud && !verifyObjectPCRUD( ctx, meta, NULL, 1 )) {
6653                         osrfAppRespondComplete( ctx, NULL );
6654                         return -1;
6655                 }
6656                 id = jsonObjectToSimpleString( jsonObjectGetIndex( ctx->params, _obj_pos ));
6657         }
6658
6659         osrfLogDebug(
6660                 OSRF_LOG_MARK,
6661                 "%s deleting %s object with %s = %s",
6662                 modulename,
6663                 osrfHashGet( meta, "fieldmapper" ),
6664                 pkey,
6665                 id
6666         );
6667
6668         jsonObject* obj = jsonNewObject( id );
6669
6670         if( strcmp( get_primitive( osrfHashGet( osrfHashGet(meta, "fields"), pkey ) ), "number" ) )
6671                 dbi_conn_quote_string( writehandle, &id );
6672
6673         dbi_result result = dbi_conn_queryf( writehandle, "DELETE FROM %s WHERE %s = %s;",
6674                 osrfHashGet( meta, "tablename" ), pkey, id );
6675
6676         int rc = 0;
6677         if( !result ) {
6678                 rc = -1;
6679                 jsonObjectFree( obj );
6680                 obj = jsonNewObject( NULL );
6681                 const char* msg;
6682                 int errnum = dbi_conn_error( writehandle, &msg );
6683                 osrfLogError(
6684                         OSRF_LOG_MARK,
6685                         "%s ERROR deleting %s object with %s = %s: %d %s",
6686                         modulename,
6687                         osrfHashGet( meta, "fieldmapper" ),
6688                         pkey,
6689                         id,
6690                         errnum,
6691                         msg ? msg : "(No description available)"
6692                 );
6693                 osrfAppSessionStatus(
6694                         ctx->session,
6695                         OSRF_STATUS_INTERNALSERVERERROR,
6696                         "osrfMethodException",
6697                         ctx->request,
6698                         "Error in deleting a row -- please see the error log for more details"
6699                 );
6700                 if( !oilsIsDBConnected( writehandle ))
6701                         osrfAppSessionPanic( ctx->session );
6702         } else
6703                 dbi_result_free( result );
6704
6705         free( id );
6706
6707         osrfAppRespondComplete( ctx, obj );
6708         jsonObjectFree( obj );
6709         return rc;
6710 }
6711
6712 /**
6713         @brief Translate a row returned from the database into a jsonObject of type JSON_ARRAY.
6714         @param result An iterator for a result set; we only look at the current row.
6715         @param @meta Pointer to the class metadata for the core class.
6716         @return Pointer to the resulting jsonObject if successful; otherwise NULL.
6717
6718         If a column is not defined in the IDL, or if it has no array_position defined for it in
6719         the IDL, or if it is defined as virtual, ignore it.
6720
6721         Otherwise, translate the column value into a jsonObject of type JSON_NULL, JSON_NUMBER,
6722         or JSON_STRING.  Then insert this jsonObject into the JSON_ARRAY according to its
6723         array_position in the IDL.
6724
6725         A field defined in the IDL but not represented in the returned row will leave a hole
6726         in the JSON_ARRAY.  In effect it will be treated as a null value.
6727
6728         In the resulting JSON_ARRAY, the field values appear in the sequence defined by the IDL,
6729         regardless of their sequence in the SELECT statement.  The JSON_ARRAY is assigned the
6730         classname corresponding to the @a meta argument.
6731
6732         The calling code is responsible for freeing the the resulting jsonObject by calling
6733         jsonObjectFree().
6734 */
6735 static jsonObject* oilsMakeFieldmapperFromResult( dbi_result result, osrfHash* meta) {
6736         if( !( result && meta )) return NULL;
6737
6738         jsonObject* object = jsonNewObjectType( JSON_ARRAY );
6739         jsonObjectSetClass( object, osrfHashGet( meta, "classname" ));
6740         osrfLogInternal( OSRF_LOG_MARK, "Setting object class to %s ", object->classname );
6741
6742         osrfHash* fields = osrfHashGet( meta, "fields" );
6743
6744         int columnIndex = 1;
6745         const char* columnName;
6746
6747         /* cycle through the columns in the row returned from the database */
6748         while( (columnName = dbi_result_get_field_name( result, columnIndex )) ) {
6749
6750                 osrfLogInternal( OSRF_LOG_MARK, "Looking for column named [%s]...", (char*) columnName );
6751
6752                 int fmIndex = -1;  // Will be set to the IDL's sequence number for this field
6753
6754                 /* determine the field type and storage attributes */
6755                 unsigned short type = dbi_result_get_field_type_idx( result, columnIndex );
6756                 int attr            = dbi_result_get_field_attribs_idx( result, columnIndex );
6757
6758                 // Fetch the IDL's sequence number for the field.  If the field isn't in the IDL,
6759                 // or if it has no sequence number there, or if it's virtual, skip it.
6760                 osrfHash* _f = osrfHashGet( fields, (char*) columnName );
6761                 if( _f ) {
6762
6763                         if( str_is_true( osrfHashGet( _f, "virtual" )))
6764                                 continue;   // skip this column: IDL says it's virtual
6765
6766                         const char* pos = (char*) osrfHashGet( _f, "array_position" );
6767                         if( !pos )      // IDL has no sequence number for it.  This shouldn't happen,
6768                                 continue;    // since we assign sequence numbers dynamically as we load the IDL.
6769
6770                         fmIndex = atoi( pos );
6771                         osrfLogInternal( OSRF_LOG_MARK, "... Found column at position [%s]...", pos );
6772                 } else {
6773                         continue;     // This field is not defined in the IDL
6774                 }
6775
6776                 // Stuff the column value into a slot in the JSON_ARRAY, indexed according to the
6777                 // sequence number from the IDL (which is likely to be different from the sequence
6778                 // of columns in the SELECT clause).
6779                 if( dbi_result_field_is_null_idx( result, columnIndex )) {
6780                         jsonObjectSetIndex( object, fmIndex, jsonNewObject( NULL ));
6781                 } else {
6782
6783                         switch( type ) {
6784
6785                                 case DBI_TYPE_INTEGER :
6786
6787                                         if( attr & DBI_INTEGER_SIZE8 )
6788                                                 jsonObjectSetIndex( object, fmIndex,
6789                                                         jsonNewNumberObject(
6790                                                                 dbi_result_get_longlong_idx( result, columnIndex )));
6791                                         else
6792                                                 jsonObjectSetIndex( object, fmIndex,
6793                                                         jsonNewNumberObject( dbi_result_get_int_idx( result, columnIndex )));
6794
6795                                         break;
6796
6797                                 case DBI_TYPE_DECIMAL :
6798                                         jsonObjectSetIndex( object, fmIndex,
6799                                                         jsonNewNumberObject( dbi_result_get_double_idx(result, columnIndex )));
6800                                         break;
6801
6802                                 case DBI_TYPE_STRING :
6803
6804                                         jsonObjectSetIndex(
6805                                                 object,
6806                                                 fmIndex,
6807                                                 jsonNewObject( dbi_result_get_string_idx( result, columnIndex ))
6808                                         );
6809
6810                                         break;
6811
6812                                 case DBI_TYPE_DATETIME : {
6813
6814                                         char dt_string[ 256 ] = "";
6815                                         struct tm gmdt;
6816
6817                                         // Fetch the date column as a time_t
6818                                         time_t _tmp_dt = dbi_result_get_datetime_idx( result, columnIndex );
6819
6820                                         // Translate the time_t to a human-readable string
6821                                         if( !( attr & DBI_DATETIME_DATE )) {
6822                                                 gmtime_r( &_tmp_dt, &gmdt );
6823                                                 strftime( dt_string, sizeof( dt_string ), "%T", &gmdt );
6824                                         } else if( !( attr & DBI_DATETIME_TIME )) {
6825                                                 gmtime_r( &_tmp_dt, &gmdt );
6826                                                 strftime( dt_string, sizeof( dt_string ), "%04Y-%m-%d", &gmdt );
6827                                         } else {
6828                                                 localtime_r( &_tmp_dt, &gmdt );
6829                                                 strftime( dt_string, sizeof( dt_string ), "%04Y-%m-%dT%T%z", &gmdt );
6830                                         }
6831
6832                                         jsonObjectSetIndex( object, fmIndex, jsonNewObject( dt_string ));
6833
6834                                         break;
6835                                 }
6836                                 case DBI_TYPE_BINARY :
6837                                         osrfLogError( OSRF_LOG_MARK,
6838                                                 "Can't do binary at column %s : index %d", columnName, columnIndex );
6839                         } // End switch
6840                 }
6841                 ++columnIndex;
6842         } // End while
6843
6844         return object;
6845 }
6846
6847 static jsonObject* oilsMakeJSONFromResult( dbi_result result ) {
6848         if( !result ) return NULL;
6849
6850         jsonObject* object = jsonNewObject( NULL );
6851
6852         time_t _tmp_dt;
6853         char dt_string[ 256 ];
6854         struct tm gmdt;
6855
6856         int columnIndex = 1;
6857         int attr;
6858         unsigned short type;
6859         const char* columnName;
6860
6861         /* cycle through the column list */
6862         while(( columnName = dbi_result_get_field_name( result, columnIndex ))) {
6863
6864                 osrfLogInternal( OSRF_LOG_MARK, "Looking for column named [%s]...", (char*) columnName );
6865
6866                 /* determine the field type and storage attributes */
6867                 type = dbi_result_get_field_type_idx( result, columnIndex );
6868                 attr = dbi_result_get_field_attribs_idx( result, columnIndex );
6869
6870                 if( dbi_result_field_is_null_idx( result, columnIndex )) {
6871                         jsonObjectSetKey( object, columnName, jsonNewObject( NULL ));
6872                 } else {
6873
6874                         switch( type ) {
6875
6876                                 case DBI_TYPE_INTEGER :
6877
6878                                         if( attr & DBI_INTEGER_SIZE8 )
6879                                                 jsonObjectSetKey( object, columnName,
6880                                                                 jsonNewNumberObject( dbi_result_get_longlong_idx(
6881                                                                                 result, columnIndex )) );
6882                                         else
6883                                                 jsonObjectSetKey( object, columnName, jsonNewNumberObject(
6884                                                                 dbi_result_get_int_idx( result, columnIndex )) );
6885                                         break;
6886
6887                                 case DBI_TYPE_DECIMAL :
6888                                         jsonObjectSetKey( object, columnName, jsonNewNumberObject(
6889                                                 dbi_result_get_double_idx( result, columnIndex )) );
6890                                         break;
6891
6892                                 case DBI_TYPE_STRING :
6893                                         jsonObjectSetKey( object, columnName,
6894                                                 jsonNewObject( dbi_result_get_string_idx( result, columnIndex )));
6895                                         break;
6896
6897                                 case DBI_TYPE_DATETIME :
6898
6899                                         memset( dt_string, '\0', sizeof( dt_string ));
6900                                         memset( &gmdt, '\0', sizeof( gmdt ));
6901
6902                                         _tmp_dt = dbi_result_get_datetime_idx( result, columnIndex );
6903
6904                                         if( !( attr & DBI_DATETIME_DATE )) {
6905                                                 gmtime_r( &_tmp_dt, &gmdt );
6906                                                 strftime( dt_string, sizeof( dt_string ), "%T", &gmdt );
6907                                         } else if( !( attr & DBI_DATETIME_TIME )) {
6908                                                 gmtime_r( &_tmp_dt, &gmdt );
6909                                                 strftime( dt_string, sizeof( dt_string ), "%04Y-%m-%d", &gmdt );
6910                                         } else {
6911                                                 localtime_r( &_tmp_dt, &gmdt );
6912                                                 strftime( dt_string, sizeof( dt_string ), "%04Y-%m-%dT%T%z", &gmdt );
6913                                         }
6914
6915                                         jsonObjectSetKey( object, columnName, jsonNewObject( dt_string ));
6916                                         break;
6917
6918                                 case DBI_TYPE_BINARY :
6919                                         osrfLogError( OSRF_LOG_MARK,
6920                                                 "Can't do binary at column %s : index %d", columnName, columnIndex );
6921                         }
6922                 }
6923                 ++columnIndex;
6924         } // end while loop traversing result
6925
6926         return object;
6927 }
6928
6929 // Interpret a string as true or false
6930 int str_is_true( const char* str ) {
6931         if( NULL == str || strcasecmp( str, "true" ) )
6932                 return 0;
6933         else
6934                 return 1;
6935 }
6936
6937 // Interpret a jsonObject as true or false
6938 static int obj_is_true( const jsonObject* obj ) {
6939         if( !obj )
6940                 return 0;
6941         else switch( obj->type )
6942         {
6943                 case JSON_BOOL :
6944                         if( obj->value.b )
6945                                 return 1;
6946                         else
6947                                 return 0;
6948                 case JSON_STRING :
6949                         if( strcasecmp( obj->value.s, "true" ) )
6950                                 return 0;
6951                         else
6952                                 return 1;
6953                 case JSON_NUMBER :          // Support 1/0 for perl's sake
6954                         if( jsonObjectGetNumber( obj ) == 1.0 )
6955                                 return 1;
6956                         else
6957                                 return 0;
6958                 default :
6959                         return 0;
6960         }
6961 }
6962
6963 // Translate a numeric code into a text string identifying a type of
6964 // jsonObject.  To be used for building error messages.
6965 static const char* json_type( int code ) {
6966         switch ( code )
6967         {
6968                 case 0 :
6969                         return "JSON_HASH";
6970                 case 1 :
6971                         return "JSON_ARRAY";
6972                 case 2 :
6973                         return "JSON_STRING";
6974                 case 3 :
6975                         return "JSON_NUMBER";
6976                 case 4 :
6977                         return "JSON_NULL";
6978                 case 5 :
6979                         return "JSON_BOOL";
6980                 default :
6981                         return "(unrecognized)";
6982         }
6983 }
6984
6985 // Extract the "primitive" attribute from an IDL field definition.
6986 // If we haven't initialized the app, then we must be running in
6987 // some kind of testbed.  In that case, default to "string".
6988 static const char* get_primitive( osrfHash* field ) {
6989         const char* s = osrfHashGet( field, "primitive" );
6990         if( !s ) {
6991                 if( child_initialized )
6992                         osrfLogError(
6993                                 OSRF_LOG_MARK,
6994                                 "%s ERROR No \"datatype\" attribute for field \"%s\"",
6995                                 modulename,
6996                                 osrfHashGet( field, "name" )
6997                         );
6998
6999                 s = "string";
7000         }
7001         return s;
7002 }
7003
7004 // Extract the "datatype" attribute from an IDL field definition.
7005 // If we haven't initialized the app, then we must be running in
7006 // some kind of testbed.  In that case, default to to NUMERIC,
7007 // since we look at the datatype only for numbers.
7008 static const char* get_datatype( osrfHash* field ) {
7009         const char* s = osrfHashGet( field, "datatype" );
7010         if( !s ) {
7011                 if( child_initialized )
7012                         osrfLogError(
7013                                 OSRF_LOG_MARK,
7014                                 "%s ERROR No \"datatype\" attribute for field \"%s\"",
7015                                 modulename,
7016                                 osrfHashGet( field, "name" )
7017                         );
7018                 else
7019                         s = "NUMERIC";
7020         }
7021         return s;
7022 }
7023
7024 /**
7025         @brief Determine whether a string is potentially a valid SQL identifier.
7026         @param s The identifier to be tested.
7027         @return 1 if the input string is potentially a valid SQL identifier, or 0 if not.
7028
7029         Purpose: to prevent certain kinds of SQL injection.  To that end we don't necessarily
7030         need to follow all the rules exactly, such as requiring that the first character not
7031         be a digit.
7032
7033         We allow leading and trailing white space.  In between, we do not allow punctuation
7034         (except for underscores and dollar signs), control characters, or embedded white space.
7035
7036         More pedantically we should allow quoted identifiers containing arbitrary characters, but
7037         for the foreseeable future such quoted identifiers are not likely to be an issue.
7038 */
7039 int is_identifier( const char* s) {
7040         if( !s )
7041                 return 0;
7042
7043         // Skip leading white space
7044         while( isspace( (unsigned char) *s ) )
7045                 ++s;
7046
7047         if( !s )
7048                 return 0;   // Nothing but white space?  Not okay.
7049
7050         // Check each character until we reach white space or
7051         // end-of-string.  Letters, digits, underscores, and
7052         // dollar signs are okay. With the exception of periods
7053         // (as in schema.identifier), control characters and other
7054         // punctuation characters are not okay.  Anything else
7055         // is okay -- it could for example be part of a multibyte
7056         // UTF8 character such as a letter with diacritical marks,
7057         // and those are allowed.
7058         do {
7059                 if( isalnum( (unsigned char) *s )
7060                         || '.' == *s
7061                         || '_' == *s
7062                         || '$' == *s )
7063                         ;  // Fine; keep going
7064                 else if(   ispunct( (unsigned char) *s )
7065                                 || iscntrl( (unsigned char) *s ) )
7066                         return 0;
7067                         ++s;
7068         } while( *s && ! isspace( (unsigned char) *s ) );
7069
7070         // If we found any white space in the above loop,
7071         // the rest had better be all white space.
7072
7073         while( isspace( (unsigned char) *s ) )
7074                 ++s;
7075
7076         if( *s )
7077                 return 0;   // White space was embedded within non-white space
7078
7079         return 1;
7080 }
7081
7082 /**
7083         @brief Determine whether to accept a character string as a comparison operator.
7084         @param op The candidate comparison operator.
7085         @return 1 if the string is acceptable as a comparison operator, or 0 if not.
7086
7087         We don't validate the operator for real.  We just make sure that it doesn't contain
7088         any semicolons or white space (with special exceptions for a few specific operators).
7089         The idea is to block certain kinds of SQL injection.  If it has no semicolons or white
7090         space but it's still not a valid operator, then the database will complain.
7091
7092         Another approach would be to compare the string against a short list of approved operators.
7093         We don't do that because we want to allow custom operators like ">100*", which at this
7094         writing would be difficult or impossible to express otherwise in a JSON query.
7095 */
7096 int is_good_operator( const char* op ) {
7097         if( !op ) return 0;   // Sanity check
7098
7099         const char* s = op;
7100         while( *s ) {
7101                 if( isspace( (unsigned char) *s ) ) {
7102                         // Special exceptions for SIMILAR TO, IS DISTINCT FROM,
7103                         // and IS NOT DISTINCT FROM.
7104                         if( !strcasecmp( op, "similar to" ) )
7105                                 return 1;
7106                         else if( !strcasecmp( op, "is distinct from" ) )
7107                                 return 1;
7108                         else if( !strcasecmp( op, "is not distinct from" ) )
7109                                 return 1;
7110                         else
7111                                 return 0;
7112                 }
7113                 else if( ';' == *s )
7114                         return 0;
7115                 ++s;
7116         }
7117         return 1;
7118 }
7119
7120 /**
7121         @name Query Frame Management
7122
7123         The following machinery supports a stack of query frames for use by SELECT().
7124
7125         A query frame caches information about one level of a SELECT query.  When we enter
7126         a subquery, we push another query frame onto the stack, and pop it off when we leave.
7127
7128         The query frame stores information about the core class, and about any joined classes
7129         in the FROM clause.
7130
7131         The main purpose is to map table aliases to classes and tables, so that a query can
7132         join to the same table more than once.  A secondary goal is to reduce the number of
7133         lookups in the IDL by caching the results.
7134 */
7135 /*@{*/
7136
7137 #define STATIC_CLASS_INFO_COUNT 3
7138
7139 static ClassInfo static_class_info[ STATIC_CLASS_INFO_COUNT ];
7140
7141 /**
7142         @brief Allocate a ClassInfo as raw memory.
7143         @return Pointer to the newly allocated ClassInfo.
7144
7145         Except for the in_use flag, which is used only by the allocation and deallocation
7146         logic, we don't initialize the ClassInfo here.
7147 */
7148 static ClassInfo* allocate_class_info( void ) {
7149         // In order to reduce the number of mallocs and frees, we return a static
7150         // instance of ClassInfo, if we can find one that we're not already using.
7151         // We rely on the fact that the compiler will implicitly initialize the
7152         // static instances so that in_use == 0.
7153
7154         int i;
7155         for( i = 0; i < STATIC_CLASS_INFO_COUNT; ++i ) {
7156                 if( ! static_class_info[ i ].in_use ) {
7157                         static_class_info[ i ].in_use = 1;
7158                         return static_class_info + i;
7159                 }
7160         }
7161
7162         // The static ones are all in use.  Malloc one.
7163
7164         return safe_malloc( sizeof( ClassInfo ) );
7165 }
7166
7167 /**
7168         @brief Free any malloc'd memory owned by a ClassInfo, returning it to a pristine state.
7169         @param info Pointer to the ClassInfo to be cleared.
7170 */
7171 static void clear_class_info( ClassInfo* info ) {
7172         // Sanity check
7173         if( ! info )
7174                 return;
7175
7176         // Free any malloc'd strings
7177
7178         if( info->alias != info->alias_store )
7179                 free( info->alias );
7180
7181         if( info->class_name != info->class_name_store )
7182                 free( info->class_name );
7183
7184         free( info->source_def );
7185
7186         info->alias = info->class_name = info->source_def = NULL;
7187         info->next = NULL;
7188 }
7189
7190 /**
7191         @brief Free a ClassInfo and everything it owns.
7192         @param info Pointer to the ClassInfo to be freed.
7193 */
7194 static void free_class_info( ClassInfo* info ) {
7195         // Sanity check
7196         if( ! info )
7197                 return;
7198
7199         clear_class_info( info );
7200
7201         // If it's one of the static instances, just mark it as not in use
7202
7203         int i;
7204         for( i = 0; i < STATIC_CLASS_INFO_COUNT; ++i ) {
7205                 if( info == static_class_info + i ) {
7206                         static_class_info[ i ].in_use = 0;
7207                         return;
7208                 }
7209         }
7210
7211         // Otherwise it must have been malloc'd, so free it
7212
7213         free( info );
7214 }
7215
7216 /**
7217         @brief Populate an already-allocated ClassInfo.
7218         @param info Pointer to the ClassInfo to be populated.
7219         @param alias Alias for the class.  If it is NULL, or an empty string, use the class
7220         name for an alias.
7221         @param class Name of the class.
7222         @return Zero if successful, or 1 if not.
7223
7224         Populate the ClassInfo with copies of the alias and class name, and with pointers to
7225         the relevant portions of the IDL for the specified class.
7226 */
7227 static int build_class_info( ClassInfo* info, const char* alias, const char* class ) {
7228         // Sanity checks
7229         if( ! info ){
7230                 osrfLogError( OSRF_LOG_MARK,
7231                                           "%s ERROR: No ClassInfo available to populate", modulename );
7232                 info->alias = info->class_name = info->source_def = NULL;
7233                 info->class_def = info->fields = info->links = NULL;
7234                 return 1;
7235         }
7236
7237         if( ! class ) {
7238                 osrfLogError( OSRF_LOG_MARK,
7239                                           "%s ERROR: No class name provided for lookup", modulename );
7240                 info->alias = info->class_name = info->source_def = NULL;
7241                 info->class_def = info->fields = info->links = NULL;
7242                 return 1;
7243         }
7244
7245         // Alias defaults to class name if not supplied
7246         if( ! alias || ! alias[ 0 ] )
7247                 alias = class;
7248
7249         // Look up class info in the IDL
7250         osrfHash* class_def = osrfHashGet( oilsIDL(), class );
7251         if( ! class_def ) {
7252                 osrfLogError( OSRF_LOG_MARK,
7253                                           "%s ERROR: Class %s not defined in IDL", modulename, class );
7254                 info->alias = info->class_name = info->source_def = NULL;
7255                 info->class_def = info->fields = info->links = NULL;
7256                 return 1;
7257         } else if( str_is_true( osrfHashGet( class_def, "virtual" ) ) ) {
7258                 osrfLogError( OSRF_LOG_MARK,
7259                                           "%s ERROR: Class %s is defined as virtual", modulename, class );
7260                 info->alias = info->class_name = info->source_def = NULL;
7261                 info->class_def = info->fields = info->links = NULL;
7262                 return 1;
7263         }
7264
7265         osrfHash* links = osrfHashGet( class_def, "links" );
7266         if( ! links ) {
7267                 osrfLogError( OSRF_LOG_MARK,
7268                                           "%s ERROR: No links defined in IDL for class %s", modulename, class );
7269                 info->alias = info->class_name = info->source_def = NULL;
7270                 info->class_def = info->fields = info->links = NULL;
7271                 return 1;
7272         }
7273
7274         osrfHash* fields = osrfHashGet( class_def, "fields" );
7275         if( ! fields ) {
7276                 osrfLogError( OSRF_LOG_MARK,
7277                                           "%s ERROR: No fields defined in IDL for class %s", modulename, class );
7278                 info->alias = info->class_name = info->source_def = NULL;
7279                 info->class_def = info->fields = info->links = NULL;
7280                 return 1;
7281         }
7282
7283         char* source_def = oilsGetRelation( class_def );
7284         if( ! source_def )
7285                 return 1;
7286
7287         // We got everything we need, so populate the ClassInfo
7288         if( strlen( alias ) > ALIAS_STORE_SIZE )
7289                 info->alias = strdup( alias );
7290         else {
7291                 strcpy( info->alias_store, alias );
7292                 info->alias = info->alias_store;
7293         }
7294
7295         if( strlen( class ) > CLASS_NAME_STORE_SIZE )
7296                 info->class_name = strdup( class );
7297         else {
7298                 strcpy( info->class_name_store, class );
7299                 info->class_name = info->class_name_store;
7300         }
7301
7302         info->source_def = source_def;
7303
7304         info->class_def = class_def;
7305         info->links     = links;
7306         info->fields    = fields;
7307
7308         return 0;
7309 }
7310
7311 #define STATIC_FRAME_COUNT 3
7312
7313 static QueryFrame static_frame[ STATIC_FRAME_COUNT ];
7314
7315 /**
7316         @brief Allocate a QueryFrame as raw memory.
7317         @return Pointer to the newly allocated QueryFrame.
7318
7319         Except for the in_use flag, which is used only by the allocation and deallocation
7320         logic, we don't initialize the QueryFrame here.
7321 */
7322 static QueryFrame* allocate_frame( void ) {
7323         // In order to reduce the number of mallocs and frees, we return a static
7324         // instance of QueryFrame, if we can find one that we're not already using.
7325         // We rely on the fact that the compiler will implicitly initialize the
7326         // static instances so that in_use == 0.
7327
7328         int i;
7329         for( i = 0; i < STATIC_FRAME_COUNT; ++i ) {
7330                 if( ! static_frame[ i ].in_use ) {
7331                         static_frame[ i ].in_use = 1;
7332                         return static_frame + i;
7333                 }
7334         }
7335
7336         // The static ones are all in use.  Malloc one.
7337
7338         return safe_malloc( sizeof( QueryFrame ) );
7339 }
7340
7341 /**
7342         @brief Free a QueryFrame, and all the memory it owns.
7343         @param frame Pointer to the QueryFrame to be freed.
7344 */
7345 static void free_query_frame( QueryFrame* frame ) {
7346         // Sanity check
7347         if( ! frame )
7348                 return;
7349
7350         clear_class_info( &frame->core );
7351
7352         // Free the join list
7353         ClassInfo* temp;
7354         ClassInfo* info = frame->join_list;
7355         while( info ) {
7356                 temp = info->next;
7357                 free_class_info( info );
7358                 info = temp;
7359         }
7360
7361         frame->join_list = NULL;
7362         frame->next = NULL;
7363
7364         // If the frame is a static instance, just mark it as unused
7365         int i;
7366         for( i = 0; i < STATIC_FRAME_COUNT; ++i ) {
7367                 if( frame == static_frame + i ) {
7368                         static_frame[ i ].in_use = 0;
7369                         return;
7370                 }
7371         }
7372
7373         // Otherwise it must have been malloc'd, so free it
7374
7375         free( frame );
7376 }
7377
7378 /**
7379         @brief Search a given QueryFrame for a specified alias.
7380         @param frame Pointer to the QueryFrame to be searched.
7381         @param target The alias for which to search.
7382         @return Pointer to the ClassInfo for the specified alias, if found; otherwise NULL.
7383 */
7384 static ClassInfo* search_alias_in_frame( QueryFrame* frame, const char* target ) {
7385         if( ! frame || ! target ) {
7386                 return NULL;
7387         }
7388
7389         ClassInfo* found_class = NULL;
7390
7391         if( !strcmp( target, frame->core.alias ) )
7392                 return &(frame->core);
7393         else {
7394                 ClassInfo* curr_class = frame->join_list;
7395                 while( curr_class ) {
7396                         if( strcmp( target, curr_class->alias ) )
7397                                 curr_class = curr_class->next;
7398                         else {
7399                                 found_class = curr_class;
7400                                 break;
7401                         }
7402                 }
7403         }
7404
7405         return found_class;
7406 }
7407
7408 /**
7409         @brief Push a new (blank) QueryFrame onto the stack.
7410 */
7411 static void push_query_frame( void ) {
7412         QueryFrame* frame = allocate_frame();
7413         frame->join_list = NULL;
7414         frame->next = curr_query;
7415
7416         // Initialize the ClassInfo for the core class
7417         ClassInfo* core = &frame->core;
7418         core->alias = core->class_name = core->source_def = NULL;
7419         core->class_def = core->fields = core->links = NULL;
7420
7421         curr_query = frame;
7422 }
7423
7424 /**
7425         @brief Pop a QueryFrame off the stack and destroy it.
7426 */
7427 static void pop_query_frame( void ) {
7428         // Sanity check
7429         if( ! curr_query )
7430                 return;
7431
7432         QueryFrame* popped = curr_query;
7433         curr_query = popped->next;
7434
7435         free_query_frame( popped );
7436 }
7437
7438 /**
7439         @brief Populate the ClassInfo for the core class.
7440         @param alias Alias for the core class.  If it is NULL or an empty string, we use the
7441         class name as an alias.
7442         @param class_name Name of the core class.
7443         @return Zero if successful, or 1 if not.
7444
7445         Populate the ClassInfo of the core class with copies of the alias and class name, and
7446         with pointers to the relevant portions of the IDL for the core class.
7447 */
7448 static int add_query_core( const char* alias, const char* class_name ) {
7449
7450         // Sanity checks
7451         if( ! curr_query ) {
7452                 osrfLogError( OSRF_LOG_MARK,
7453                                           "%s ERROR: No QueryFrame available for class %s", modulename, class_name );
7454                 return 1;
7455         } else if( curr_query->core.alias ) {
7456                 osrfLogError( OSRF_LOG_MARK,
7457                                           "%s ERROR: Core class %s already populated as %s",
7458                                           modulename, curr_query->core.class_name, curr_query->core.alias );
7459                 return 1;
7460         }
7461
7462         build_class_info( &curr_query->core, alias, class_name );
7463         if( curr_query->core.alias )
7464                 return 0;
7465         else {
7466                 osrfLogError( OSRF_LOG_MARK,
7467                                           "%s ERROR: Unable to look up core class %s", modulename, class_name );
7468                 return 1;
7469         }
7470 }
7471
7472 /**
7473         @brief Search the current QueryFrame for a specified alias.
7474         @param target The alias for which to search.
7475         @return A pointer to the corresponding ClassInfo, if found; otherwise NULL.
7476 */
7477 static inline ClassInfo* search_alias( const char* target ) {
7478         return search_alias_in_frame( curr_query, target );
7479 }
7480
7481 /**
7482         @brief Search all levels of query for a specified alias, starting with the current query.
7483         @param target The alias for which to search.
7484         @return A pointer to the corresponding ClassInfo, if found; otherwise NULL.
7485 */
7486 static ClassInfo* search_all_alias( const char* target ) {
7487         ClassInfo* found_class = NULL;
7488         QueryFrame* curr_frame = curr_query;
7489
7490         while( curr_frame ) {
7491                 if(( found_class = search_alias_in_frame( curr_frame, target ) ))
7492                         break;
7493                 else
7494                         curr_frame = curr_frame->next;
7495         }
7496
7497         return found_class;
7498 }
7499
7500 /**
7501         @brief Add a class to the list of classes joined to the current query.
7502         @param alias Alias of the class to be added.  If it is NULL or an empty string, we use
7503         the class name as an alias.
7504         @param classname The name of the class to be added.
7505         @return A pointer to the ClassInfo for the added class, if successful; otherwise NULL.
7506 */
7507 static ClassInfo* add_joined_class( const char* alias, const char* classname ) {
7508
7509         if( ! classname || ! *classname ) {    // sanity check
7510                 osrfLogError( OSRF_LOG_MARK, "Can't join a class with no class name" );
7511                 return NULL;
7512         }
7513
7514         if( ! alias )
7515                 alias = classname;
7516
7517         const ClassInfo* conflict = search_alias( alias );
7518         if( conflict ) {
7519                 osrfLogError( OSRF_LOG_MARK,
7520                                           "%s ERROR: Table alias \"%s\" conflicts with class \"%s\"",
7521                                           modulename, alias, conflict->class_name );
7522                 return NULL;
7523         }
7524
7525         ClassInfo* info = allocate_class_info();
7526
7527         if( build_class_info( info, alias, classname ) ) {
7528                 free_class_info( info );
7529                 return NULL;
7530         }
7531
7532         // Add the new ClassInfo to the join list of the current QueryFrame
7533         info->next = curr_query->join_list;
7534         curr_query->join_list = info;
7535
7536         return info;
7537 }
7538
7539 /**
7540         @brief Destroy all nodes on the query stack.
7541 */
7542 static void clear_query_stack( void ) {
7543         while( curr_query )
7544                 pop_query_frame();
7545 }
7546
7547 /**
7548         @brief Implement the set_audit_info method.
7549         @param ctx Pointer to the method context.
7550         @return Zero if successful, or -1 if not.
7551
7552         Issue a SAVEPOINT to the database server.
7553
7554         Method parameters:
7555         - authkey
7556         - user id (int)
7557         - workstation id (int)
7558
7559         If user id is not provided the authkey will be used.
7560         For PCRUD the authkey is always used, even if a user is provided.
7561 */
7562 int setAuditInfo( osrfMethodContext* ctx ) {
7563         if(osrfMethodVerifyContext( ctx )) {
7564                 osrfLogError( OSRF_LOG_MARK,  "Invalid method context" );
7565                 return -1;
7566         }
7567
7568         // Get the user id from the parameters
7569         const char* user_id = jsonObjectGetString( jsonObjectGetIndex(ctx->params, 1) );
7570
7571         if( enforce_pcrud || !user_id ) {
7572                 timeout_needs_resetting = 1;
7573                 const jsonObject* user = verifyUserPCRUD( ctx );
7574                 if( !user )
7575                         return -1;
7576                 osrfAppRespondComplete( ctx, NULL );
7577                 return 0;
7578         }
7579
7580         // Not PCRUD and have a user_id?
7581         int result = writeAuditInfo( ctx, user_id, jsonObjectGetString( jsonObjectGetIndex(ctx->params, 2) ) );
7582         osrfAppRespondComplete( ctx, NULL );
7583         return result;
7584 }
7585
7586 /**
7587         @brief Save a audit info
7588         @param ctx Pointer to the method context.
7589         @param user_id User ID to write as a string
7590         @param ws_id Workstation ID to write as a string
7591 */
7592 int writeAuditInfo( osrfMethodContext* ctx, const char* user_id, const char* ws_id) {
7593         if( ctx && ctx->session ) {
7594                 osrfAppSession* session = ctx->session;
7595
7596                 osrfHash* cache = session->userData;
7597
7598                 // If the session doesn't already have a hash, create one.  Make sure
7599                 // that the application session frees the hash when it terminates.
7600                 if( NULL == cache ) {
7601                         session->userData = cache = osrfNewHash();
7602                         osrfHashSetCallback( cache, &sessionDataFree );
7603                         ctx->session->userDataFree = &userDataFree;
7604                 }
7605
7606                 dbi_result result = dbi_conn_queryf( writehandle, "SELECT auditor.set_audit_info( %s, %s );", user_id, ws_id ? ws_id : "NULL" );
7607                 if( !result ) {
7608                         osrfLogWarning( OSRF_LOG_MARK, "BAD RESULT" );
7609                         const char* msg;
7610                         int errnum = dbi_conn_error( writehandle, &msg );
7611                         osrfLogError(
7612                                 OSRF_LOG_MARK,
7613                                 "%s: Error setting auditor information: %d %s",
7614                                 modulename,
7615                                 errnum,
7616                                 msg ? msg : "(No description available)"
7617                         );
7618                         osrfAppSessionStatus( ctx->session, OSRF_STATUS_INTERNALSERVERERROR,
7619                                 "osrfMethodException", ctx->request, "Error setting auditor info" );
7620                         if( !oilsIsDBConnected( writehandle ))
7621                                 osrfAppSessionPanic( ctx->session );
7622                         return -1;
7623                 } else {
7624                         dbi_result_free( result );
7625                 }
7626         }
7627         return 0;
7628 }
7629
7630 /**
7631         @brief Remove all but safe character from savepoint name
7632         @param sp User-supplied savepoint name
7633         @return sanitized savepoint name, or NULL
7634
7635     The caller is expected to free the returned string.  Note that
7636     this function exists only because we can't use PQescapeLiteral
7637     without either forking libdbi or abandoning it.
7638 */
7639 static char* _sanitize_savepoint_name( const char* sp ) {
7640
7641         const char* safe_chars = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ012345789_";
7642
7643         // PostgreSQL uses NAMEDATALEN-1 as a max length for identifiers,
7644         // and the default value of NAMEDATALEN is 64; that should be long enough
7645         // for our purposes, and it's unlikely that anyone is going to recompile
7646         // PostgreSQL to have a smaller value, so cap the identifier name
7647         // accordingly to avoid the remote chance that someone manages to pass in a
7648         // 12GB savepoint name
7649         const int MAX_LITERAL_NAMELEN = 63;
7650         int len = 0;
7651         len = strlen( sp );
7652         if (len > MAX_LITERAL_NAMELEN) {
7653                 len = MAX_LITERAL_NAMELEN;
7654         }
7655
7656         char* safeSpName = safe_malloc( len + 1 );
7657         int i = 0;
7658         int j;
7659         char* found;
7660         for (j = 0; j < len; j++) {
7661         found = strchr(safe_chars, sp[j]);
7662                 if (found) {
7663                         safeSpName[ i++ ] = found[0];
7664                 }
7665         }
7666         safeSpName[ i ] = '\0';
7667         return safeSpName;
7668 }
7669
7670 /**
7671         @brief Remove all but safe character from TZ name
7672         @param tz User-supplied TZ name
7673         @return sanitized TZ name, or NULL
7674
7675     The caller is expected to free the returned string.  Note that
7676     this function exists only because we can't use PQescapeLiteral
7677     without either forking libdbi or abandoning it.
7678 */
7679 static char* _sanitize_tz_name( const char* tz ) {
7680
7681         if (NULL == tz) return NULL;
7682
7683         const char* safe_chars = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ012345789_/-+";
7684
7685         // PostgreSQL uses NAMEDATALEN-1 as a max length for identifiers,
7686         // and the default value of NAMEDATALEN is 64; that should be long enough
7687         // for our purposes, and it's unlikely that anyone is going to recompile
7688         // PostgreSQL to have a smaller value, so cap the identifier name
7689         // accordingly to avoid the remote chance that someone manages to pass in a
7690         // 12GB savepoint name
7691         const int MAX_LITERAL_NAMELEN = 63;
7692         int len = 0;
7693         len = strlen( tz );
7694         if (len > MAX_LITERAL_NAMELEN) {
7695                 len = MAX_LITERAL_NAMELEN;
7696         }
7697
7698         char* safeSpName = safe_malloc( len + 1 );
7699         int i = 0;
7700         int j;
7701         char* found;
7702         for (j = 0; j < len; j++) {
7703         found = strchr(safe_chars, tz[j]);
7704                 if (found) {
7705                         safeSpName[ i++ ] = found[0];
7706                 }
7707         }
7708         safeSpName[ i ] = '\0';
7709         return safeSpName;
7710 }
7711
7712 /*@}*/