Index: TODO =================================================================== diff -u -rcd33e8cefca1d52063ebcb6689e46527bb94e33d -r7543d1df847248f723f02fa1abc6645713b9d10f --- TODO (.../TODO) (revision cd33e8cefca1d52063ebcb6689e46527bb94e33d) +++ TODO (.../TODO) (revision 7543d1df847248f723f02fa1abc6645713b9d10f) @@ -4909,6 +4909,16 @@ nx::traits: - handle ensemble methods correctly (use full method path for resolution) - add new regression tests for traits + +nx-mongo: +- optional support for mongodb connection pools (compile time macro + USE_CLIENT_POOL controls this, per default, this is on) +- allow to pass "-metadata" to gridfile::create to ease metadata attachment + to gridfiles +- some conveniance improvements in nx-mongo.tcl (ability to ignore + attributes in "bson encode", ability to unset attributes in gridfs, ...) +- bump version numbers of nsfmongo to 0.3 and nx-monogo to 0.5 + ======================================================================== TODO: Index: library/mongodb/configure.ac =================================================================== diff -u -rcef3de5c4f65e767d0c66389bacc77bc3c2e5a68 -r7543d1df847248f723f02fa1abc6645713b9d10f --- library/mongodb/configure.ac (.../configure.ac) (revision cef3de5c4f65e767d0c66389bacc77bc3c2e5a68) +++ library/mongodb/configure.ac (.../configure.ac) (revision 7543d1df847248f723f02fa1abc6645713b9d10f) @@ -20,7 +20,7 @@ # so you can encode the package version directly into the source files. #----------------------------------------------------------------------- -AC_INIT([nsfmongo], [0.2]) +AC_INIT([nsfmongo], [0.3]) #-------------------------------------------------------------------- # Call TEA_INIT as the first TEA_ macro to set up initial vars. Index: library/mongodb/mongoAPI.decls =================================================================== diff -u -rcef3de5c4f65e767d0c66389bacc77bc3c2e5a68 -r7543d1df847248f723f02fa1abc6645713b9d10f --- library/mongodb/mongoAPI.decls (.../mongoAPI.decls) (revision cef3de5c4f65e767d0c66389bacc77bc3c2e5a68) +++ library/mongodb/mongoAPI.decls (.../mongoAPI.decls) (revision 7543d1df847248f723f02fa1abc6645713b9d10f) @@ -122,6 +122,7 @@ {-argName "value" -required 1} {-argName "name" -required 1} {-argName "contenttype" -required 1} + {-argName "-metadata" -type tclobj} } cmd "gridfile::delete" NsfMongoGridFileDelete { Index: library/mongodb/mongoAPI.h =================================================================== diff -u -rcef3de5c4f65e767d0c66389bacc77bc3c2e5a68 -r7543d1df847248f723f02fa1abc6645713b9d10f --- library/mongodb/mongoAPI.h (.../mongoAPI.h) (revision cef3de5c4f65e767d0c66389bacc77bc3c2e5a68) +++ library/mongodb/mongoAPI.h (.../mongoAPI.h) (revision 7543d1df847248f723f02fa1abc6645713b9d10f) @@ -132,7 +132,7 @@ static int NsfMongoGridFSClose(Tcl_Interp *interp, mongoc_gridfs_t *gfsPtr, Tcl_Obj *gfsObj); static int NsfMongoGridFSOpen(Tcl_Interp *interp, mongoc_client_t *connPtr, CONST char *dbname, CONST char *prefix); static int NsfMongoGridFileClose(Tcl_Interp *interp, mongoc_gridfs_file_t *gridfilePtr, Tcl_Obj *gridfileObj); -static int NsfMongoGridFileCreate(Tcl_Interp *interp, int withSource, mongoc_gridfs_t *gfsPtr, CONST char *value, CONST char *name, CONST char *contenttype); +static int NsfMongoGridFileCreate(Tcl_Interp *interp, int withSource, mongoc_gridfs_t *gfsPtr, CONST char *value, CONST char *name, CONST char *contenttype, Tcl_Obj *withMetadata); static int NsfMongoGridFileDelete(Tcl_Interp *interp, mongoc_gridfs_t *gfsPtr, Tcl_Obj *query); static int NsfMongoGridFileGetContentType(Tcl_Interp *interp, mongoc_gridfs_file_t *gridfilePtr); static int NsfMongoGridFileGetContentlength(Tcl_Interp *interp, mongoc_gridfs_file_t *gridfilePtr); @@ -517,9 +517,10 @@ CONST char *value = (CONST char *)pc.clientData[2]; CONST char *name = (CONST char *)pc.clientData[3]; CONST char *contenttype = (CONST char *)pc.clientData[4]; + Tcl_Obj *withMetadata = (Tcl_Obj *)pc.clientData[5]; assert(pc.status == 0); - return NsfMongoGridFileCreate(interp, withSource, gfsPtr, value, name, contenttype); + return NsfMongoGridFileCreate(interp, withSource, gfsPtr, value, name, contenttype, withMetadata); } else { return TCL_ERROR; @@ -762,12 +763,13 @@ {"::mongo::gridfile::close", NsfMongoGridFileCloseStub, 1, { {"gridfile", NSF_ARG_REQUIRED, 1, Nsf_ConvertTo_Pointer, NULL,NULL,"mongoc_gridfs_file_t",NULL,NULL,NULL,NULL,NULL}} }, -{"::mongo::gridfile::create", NsfMongoGridFileCreateStub, 5, { +{"::mongo::gridfile::create", NsfMongoGridFileCreateStub, 6, { {"-source", NSF_ARG_REQUIRED|NSF_ARG_IS_ENUMERATION, 1, ConvertToGridfilesource, NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL}, {"gfs", NSF_ARG_REQUIRED, 1, Nsf_ConvertTo_Pointer, NULL,NULL,"mongoc_gridfs_t",NULL,NULL,NULL,NULL,NULL}, {"value", NSF_ARG_REQUIRED, 1, Nsf_ConvertTo_String, NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL}, {"name", NSF_ARG_REQUIRED, 1, Nsf_ConvertTo_String, NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL}, - {"contenttype", NSF_ARG_REQUIRED, 1, Nsf_ConvertTo_String, NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL}} + {"contenttype", NSF_ARG_REQUIRED, 1, Nsf_ConvertTo_String, NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL}, + {"-metadata", 0, 1, Nsf_ConvertTo_Tclobj, NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL}} }, {"::mongo::gridfile::delete", NsfMongoGridFileDeleteStub, 2, { {"gfs", NSF_ARG_REQUIRED, 1, Nsf_ConvertTo_Pointer, NULL,NULL,"mongoc_gridfs_t",NULL,NULL,NULL,NULL,NULL}, Index: library/mongodb/nsfmongo.c =================================================================== diff -u -r0067fc15903b586ffad2a459ce1cf7294c8158d3 -r7543d1df847248f723f02fa1abc6645713b9d10f --- library/mongodb/nsfmongo.c (.../nsfmongo.c) (revision 0067fc15903b586ffad2a459ce1cf7294c8158d3) +++ library/mongodb/nsfmongo.c (.../nsfmongo.c) (revision 7543d1df847248f723f02fa1abc6645713b9d10f) @@ -24,6 +24,8 @@ #include #include +#define USE_CLIENT_POOL 1 + /* * Define the counters to generate nice symbols for pointer converter */ @@ -33,6 +35,13 @@ static int mongoCollectionCount = 0; static int mongoCursorCount = 0; +#if defined(USE_CLIENT_POOL) +static NsfMutex poolMutex = 0; +static mongoc_client_pool_t *mongoClientPool = NULL; +static int mongoClientPoolRefCount = 0; +static mongoc_uri_t *mongoUri = NULL; +#endif + typedef enum { NSF_BSON_ARRAY, NSF_BSON_BOOL, @@ -57,7 +66,7 @@ "boolean", "int32", "int64", - "date_time", + "datetime", "document", "double", "minkey", @@ -193,7 +202,8 @@ case BSON_TYPE_BOOL: tag = NSF_BSON_BOOL; elemObj = Tcl_NewBooleanObj(bson_iter_bool( &i )); break; case BSON_TYPE_REGEX: { const char *options = NULL; /* TODO: not handled */ - tag = NSF_BSON_REGEX; elemObj = Tcl_NewStringObj(bson_iter_regex( &i, &options ), -1); + tag = NSF_BSON_REGEX; + elemObj = Tcl_NewStringObj(bson_iter_regex( &i, &options ), -1); break; } case BSON_TYPE_UTF8: { @@ -501,7 +511,11 @@ NsfMongoClose(Tcl_Interp *interp, mongoc_client_t *clientPtr, Tcl_Obj *clientObj) { if (clientPtr) { +#if defined(USE_CLIENT_POOL) + mongoc_client_pool_push(mongoClientPool, clientPtr); +#else mongoc_client_destroy(clientPtr); +#endif Nsf_PointerDelete(ObjStr(clientObj), clientPtr, 0); } return TCL_OK; @@ -521,7 +535,21 @@ if (uri == NULL) { uri = "mongodb://127.0.0.1:27017/"; } + +#if defined(USE_CLIENT_POOL) + NsfMutexLock(&poolMutex); + + if (mongoClientPool == NULL) { + mongoUri = mongoc_uri_new(uri); + NsfLog(interp, NSF_LOG_NOTICE, "nsf::mongo::connect: creating pool with uri %s", uri); + mongoClientPool = mongoc_client_pool_new(mongoUri); + } + + NsfMutexUnlock(&poolMutex); + clientPtr = mongoc_client_pool_pop(mongoClientPool); +#else clientPtr = mongoc_client_new(uri); +#endif if (clientPtr == NULL) { return NsfPrintError(interp, "failed to parse Mongo URI"); @@ -1118,29 +1146,42 @@ {-argName "value" -required 1} {-argName "name" -required 1} {-argName "contenttype" -required 1} + {-argName "-metadata" -required 0 -nrags 1 -type tclobj} } */ static int NsfMongoGridFileCreate(Tcl_Interp *interp, int withSource, mongoc_gridfs_t *gridfsPtr, CONST char *value, CONST char *name, - CONST char *contenttype) { + CONST char *contenttype, + Tcl_Obj *withMetadata + ) { int result = TCL_OK; mongoc_gridfs_file_opt_t fileOpts = {NULL}; mongoc_gridfs_file_t *gridFile; + bson_t bsonMetaData[1]; if (withSource == GridfilesourceNULL) { withSource = GridfilesourceFileIdx; } + if (withMetadata != NULL) { + Tcl_Obj **objv; + int objc; + + result = Tcl_ListObjGetElements(interp, withMetadata, &objc, &objv); + if (result != TCL_OK || (objc % 3 != 0)) { + return NsfPrintError(interp, "%s: must contain a multiple of 3 elements", ObjStr(withMetadata)); + } + BsonAppendObjv(interp, bsonMetaData, objc, objv); + fileOpts.metadata = bsonMetaData; + } + fileOpts.filename = name; fileOpts.content_type = contenttype; /* const char *md5; - const char *filename; - const char *content_type; const bson_t *aliases; - const bson_t *metadata; uint32_t chunk_size; */ gridFile = mongoc_gridfs_create_file(gridfsPtr, &fileOpts); @@ -1337,16 +1378,30 @@ return TCL_OK; } +#if 0 +bson_t * +mongoc_gridfs_file_get_metadata(mongoc_gridfs_file_t *file) { + return &file->bson_metadata; +} + +void +mongoc_gridfs_file_set_metadata(mongoc_gridfs_file_t *file, bson_t *metadata) { + file->bson_metadata = *metadata; +} +#endif + /* cmd gridfile::get_metadata NsfMongoGridFileGetMetaData { {-argName "gridfile" -required 1 -type mongoc_gridfs_file_t} } */ static int NsfMongoGridFileGetMetaData(Tcl_Interp *interp, mongoc_gridfs_file_t* gridFilePtr) { + const bson_t *metaDataPtr = mongoc_gridfs_file_get_metadata(gridFilePtr); - Tcl_SetObjResult(interp, BsonToList(interp, &gridFilePtr->bson_metadata, 0)); - + if (metaDataPtr != NULL) { + Tcl_SetObjResult(interp, BsonToList(interp, metaDataPtr, 0)); + } return TCL_OK; } @@ -1395,6 +1450,29 @@ ***********************************************************************/ void +Nsfmongo_ThreadExit(ClientData clientData) { + /* + * The exit might happen at a time, when tcl is already shut down. + * We can't reliably call NsfLog. + */ + fprintf(stderr, "+++ Nsfmongo_ThreadExit\n"); +#if defined(USE_CLIENT_POOL) + NsfMutexLock(&poolMutex); + mongoClientPoolRefCount --; + if (mongoClientPool != NULL) { + fprintf(stderr, "========= Nsfmongo_ThreadExit mongoClientPoolRefCount %d\n", mongoClientPoolRefCount); + if (mongoClientPoolRefCount < 1) { + mongoc_client_pool_destroy(mongoClientPool); + mongoClientPool = NULL; + mongoc_uri_destroy(mongoUri); + mongoUri = NULL; + } + } + NsfMutexUnlock(&poolMutex); +#endif +} + +void Nsfmongo_Exit(ClientData clientData) { /* * The exit might happen at a time, when tcl is already shut down. @@ -1403,80 +1481,96 @@ * Tcl_Interp *interp = (Tcl_Interp *)clientData; * NsfLog(interp,NSF_LOG_NOTICE, "Nsfmongo Exit"); */ + fprintf(stderr, "+++ Nsfmongo_Exit\n"); +#if defined(TCL_THREADS) + Tcl_DeleteThreadExitHandler(Nsfmongo_ThreadExit, clientData); +#endif + Tcl_Release(clientData); } + + extern int Nsfmongo_Init(Tcl_Interp * interp) { int i; static NsfMutex initMutex = 0; - #ifdef USE_TCL_STUBS - if (Tcl_InitStubs(interp, TCL_VERSION, 0) == NULL) { - return TCL_ERROR; - } + if (Tcl_InitStubs(interp, TCL_VERSION, 0) == NULL) { + return TCL_ERROR; + } # ifdef USE_NSF_STUBS - if (Nsf_InitStubs(interp, "2.0", 0) == NULL) { - return TCL_ERROR; - } + if (Nsf_InitStubs(interp, "2.0", 0) == NULL) { + return TCL_ERROR; + } # endif #else - if (Tcl_PkgRequire(interp, "Tcl", TCL_VERSION, 0) == NULL) { - return TCL_ERROR; - } + if (Tcl_PkgRequire(interp, "Tcl", TCL_VERSION, 0) == NULL) { + return TCL_ERROR; + } #endif - Tcl_PkgProvide(interp, "nsf::mongo", PACKAGE_VERSION); + Tcl_PkgProvide(interp, "nsf::mongo", PACKAGE_VERSION); #ifdef PACKAGE_REQUIRE_FROM_SLAVE_INTERP_WORKS_NOW - if (Tcl_PkgRequire(interp, "nsf", PACKAGE_VERSION, 0) == NULL) { - return TCL_ERROR; - } + if (Tcl_PkgRequire(interp, "nsf", PACKAGE_VERSION, 0) == NULL) { + return TCL_ERROR; + } #endif - Tcl_CreateExitHandler(Nsfmongo_Exit, interp); + Tcl_Preserve(interp); +#if defined(TCL_THREADS) + Tcl_CreateThreadExitHandler(Nsfmongo_ThreadExit, interp); +#endif + Tcl_CreateExitHandler(Nsfmongo_Exit, interp); - /* - * Register global mongo tcl_objs - */ - NsfMutexLock(&initMutex); - if (NsfMongoGlobalObjs == NULL) { - NsfMongoGlobalObjs = (Tcl_Obj **)ckalloc(sizeof(Tcl_Obj*)*nr_elements(NsfMongoGlobalStrings)); - for (i = 0; i < nr_elements(NsfMongoGlobalStrings); i++) { - NsfMongoGlobalObjs[i] = Tcl_NewStringObj(NsfMongoGlobalStrings[i], -1); - Tcl_IncrRefCount(NsfMongoGlobalObjs[i]); - } +#if defined(USE_CLIENT_POOL) + NsfMutexLock(&poolMutex); + mongoClientPoolRefCount ++; + NsfMutexUnlock(&poolMutex); +#endif + + /* + * Register global mongo tcl_objs + */ + NsfMutexLock(&initMutex); + if (NsfMongoGlobalObjs == NULL) { + NsfMongoGlobalObjs = (Tcl_Obj **)ckalloc(sizeof(Tcl_Obj*)*nr_elements(NsfMongoGlobalStrings)); + for (i = 0; i < nr_elements(NsfMongoGlobalStrings); i++) { + NsfMongoGlobalObjs[i] = Tcl_NewStringObj(NsfMongoGlobalStrings[i], -1); + Tcl_IncrRefCount(NsfMongoGlobalObjs[i]); } - NsfMutexUnlock(&initMutex); + } + NsfMutexUnlock(&initMutex); - /* - * register the pointer converter - */ - Nsf_PointerTypeRegister(interp, "mongoc_client_t", &mongoClientCount); - Nsf_PointerTypeRegister(interp, "mongoc_collection_t", &mongoCollectionCount); - Nsf_PointerTypeRegister(interp, "mongoc_cursor_t", &mongoCursorCount); - Nsf_PointerTypeRegister(interp, "mongoc_gridfs_file_t", &gridfileCount); - Nsf_PointerTypeRegister(interp, "mongoc_gridfs_t", &gridfsCount); + /* + * register the pointer converter + */ + Nsf_PointerTypeRegister(interp, "mongoc_client_t", &mongoClientCount); + Nsf_PointerTypeRegister(interp, "mongoc_collection_t", &mongoCollectionCount); + Nsf_PointerTypeRegister(interp, "mongoc_cursor_t", &mongoCursorCount); + Nsf_PointerTypeRegister(interp, "mongoc_gridfs_file_t", &gridfileCount); + Nsf_PointerTypeRegister(interp, "mongoc_gridfs_t", &gridfsCount); - for (i=0; i < nr_elements(method_command_namespace_names); i++) { - Tcl_CreateNamespace(interp, method_command_namespace_names[i], 0, (Tcl_NamespaceDeleteProc *)NULL); - } + for (i=0; i < nr_elements(method_command_namespace_names); i++) { + Tcl_CreateNamespace(interp, method_command_namespace_names[i], 0, (Tcl_NamespaceDeleteProc *)NULL); + } - /* create all method commands (will use the namespaces above) */ - for (i=0; i < nr_elements(method_definitions)-1; i++) { - Tcl_CreateObjCommand(interp, method_definitions[i].methodName, method_definitions[i].proc, 0, 0); - } + /* create all method commands (will use the namespaces above) */ + for (i=0; i < nr_elements(method_definitions)-1; i++) { + Tcl_CreateObjCommand(interp, method_definitions[i].methodName, method_definitions[i].proc, 0, 0); + } - Tcl_SetIntObj(Tcl_GetObjResult(interp), 1); - return TCL_OK; + Tcl_SetIntObj(Tcl_GetObjResult(interp), 1); + return TCL_OK; } extern int Nsfmongo_SafeInit(interp) - Tcl_Interp *interp; + Tcl_Interp *interp; { - return Nsfmongo_Init(interp); + return Nsfmongo_Init(interp); } /* Index: library/mongodb/nx-mongo.tcl =================================================================== diff -u -r0067fc15903b586ffad2a459ce1cf7294c8158d3 -r7543d1df847248f723f02fa1abc6645713b9d10f --- library/mongodb/nx-mongo.tcl (.../nx-mongo.tcl) (revision 0067fc15903b586ffad2a459ce1cf7294c8158d3) +++ library/mongodb/nx-mongo.tcl (.../nx-mongo.tcl) (revision 7543d1df847248f723f02fa1abc6645713b9d10f) @@ -5,7 +5,7 @@ # package require nx package require nsf::mongo -package provide nx::mongo 0.4 +package provide nx::mongo 0.5 # todo: how to handle multiple connections; currently we have a single, global connection # todo: all references are currently auto-fetched. make this optional @@ -103,8 +103,9 @@ unset :gridFs :gridFsName } - :public object method "gridfs create" {{-source file} value name {mime text/plain}} { - ::mongo::gridfile::create -source $source ${:gridFs} $value $name $mime + :public object method "gridfs create" {{-source file} value name {mime text/plain} {-metadata}} { + ::mongo::gridfile::create -source $source ${:gridFs} $value $name $mime \ + {*}[expr {[info exists metadata] ? [list -metadata $metadata] : {}}] } :public object method "gridfs list" {{-all:switch false} query} { @@ -160,8 +161,22 @@ nx::mongo::db gridfs update [dict get $d _id] $bson } + :public object method "gridfs unset attribute" {query attribute} { + set info [::nx::mongo::db gridfs list $query] + if {$info eq ""} {error "no such file <$query> stored in gridfs"} + foreach {att type v} $info { dict set d $att $v } + if {[dict exists $d $attribute]} { + # delete the attribute + nx::mongo::db gridfs update [dict get $d _id] [list \$unset document [list $attribute string ""]] + } else { + # does not exist, nothing to do + } + } + :public object method "gridfs map" {query url} { - ::nx::mongo::db gridfs set attribute $query url $url + # map always the newest entry + set fullQuery [list \$query document $query \$orderby document {uploadDate integer -1}] + ::nx::mongo::db gridfs set attribute $fullQuery url $url } :public object method "gridfs mapped" {url} { set info [::mongo::collection::query [:collection ${:db}.${:gridFsName}.files] \ @@ -354,6 +369,16 @@ error "value '$value' for property $name is not of type $arg" } } + # + # Type converter for datetime handling (scan date strings into + # input values into integers in the form mongo expects it) + # + :public method type=datetime {name value} { + # puts stderr "... [clock format [clock scan $value -format {%B %d, %Y}] -format {%B %d, %Y}]" + # MongoDB stores time in ms + if {[info exists :scanformat]} {return [expr {[clock scan $value -format ${:scanformat}] * 1000}]} + return [expr {[clock scan $value] * 1000}] + } } @@ -529,7 +554,7 @@ spec:parameter {initblock ""} } { - regsub -all {,type=} $spec {,arg=} spec + regsub -all {,type=::} $spec {,arg=::} spec set result [next [list -accessor $accessor -class $class \ -configurable $configurable -incremental=$incremental \ $spec $initblock]] @@ -548,7 +573,7 @@ spec:parameter defaultValue:optional } { - regsub -all {,type=} $spec {,arg=} spec + regsub -all {,type=::} $spec {,arg=::} spec set result [next [list -accessor $accessor -class $class \ -configurable $configurable -incremental=$incremental \ -initblock $initblock $spec \ @@ -788,17 +813,18 @@ # # _id is the special property maintained by mongoDB # - :property -class ::nx::mongo::Attribute _id { + :property -accessor public -class ::nx::mongo::Attribute _id { set :mongotype oid } # # Encode all object data in bson notation # - :method "bson encode" {} { + :method "bson encode" {{-ignore ""}} { set bson [list] set cls [:info class] foreach var [:info vars] { + if {$var in $ignore} continue set slot [$cls get slot $var] if {$slot ne ""} { if {[nx::var exists $slot rep] && [nx::var set $slot rep] ne ""} {