Index: openacs-4/packages/acs-content-repository/tcl/acs-content-repository-init-oracle.xql =================================================================== RCS file: /usr/local/cvsroot/openacs-4/packages/acs-content-repository/tcl/acs-content-repository-init-oracle.xql,v diff -u -N --- openacs-4/packages/acs-content-repository/tcl/acs-content-repository-init-oracle.xql 13 May 2001 05:21:45 -0000 1.1 +++ /dev/null 1 Jan 1970 00:00:00 -0000 @@ -1,15 +0,0 @@ - - - oracle8.1.6 - - - - - begin - cr_scheduled_release_exec; - end; - - - - - Index: openacs-4/packages/acs-content-repository/tcl/acs-content-repository-init-postgresql.xql =================================================================== RCS file: /usr/local/cvsroot/openacs-4/packages/acs-content-repository/tcl/acs-content-repository-init-postgresql.xql,v diff -u -N --- openacs-4/packages/acs-content-repository/tcl/acs-content-repository-init-postgresql.xql 13 May 2001 05:21:45 -0000 1.1 +++ /dev/null 1 Jan 1970 00:00:00 -0000 @@ -1,13 +0,0 @@ - - - postgresql7.1 - - - - - select cr_scheduled_release_exec(); - - - - - Index: openacs-4/packages/acs-content-repository/tcl/acs-content-repository-init.tcl =================================================================== RCS file: /usr/local/cvsroot/openacs-4/packages/acs-content-repository/tcl/acs-content-repository-init.tcl,v diff -u -N -r1.18 -r1.18.2.1 --- openacs-4/packages/acs-content-repository/tcl/acs-content-repository-init.tcl 17 May 2018 14:17:35 -0000 1.18 +++ openacs-4/packages/acs-content-repository/tcl/acs-content-repository-init.tcl 22 Feb 2021 11:47:19 -0000 1.18.2.1 @@ -13,17 +13,7 @@ ad_schedule_proc -thread t -schedule_proc ns_schedule_daily [list 22 0] cr_delete_scheduled_files -ad_proc -public acs_cr_scheduled_release_exec {} { - This was handled by oracle, but since other dbs, such as PostgreSQL don't - support job submission, the job scheduling has been moved to aolserver. - (OpenACS - DanW) - -} { - - db_exec_plsql schedule_releases {} -} - ad_schedule_proc [expr {15 * 60}] acs_cr_scheduled_release_exec nsv_set CR_LOCATIONS . "" Index: openacs-4/packages/acs-content-repository/tcl/acs-content-repository-procs-oracle.xql =================================================================== RCS file: /usr/local/cvsroot/openacs-4/packages/acs-content-repository/tcl/acs-content-repository-procs-oracle.xql,v diff -u -N -r1.6 -r1.6.2.1 --- openacs-4/packages/acs-content-repository/tcl/acs-content-repository-procs-oracle.xql 7 Aug 2017 23:47:47 -0000 1.6 +++ openacs-4/packages/acs-content-repository/tcl/acs-content-repository-procs-oracle.xql 22 Feb 2021 11:47:19 -0000 1.6.2.1 @@ -5,12 +5,21 @@ -select distinct crftd.path, crftd.storage_area_key - from cr_files_to_delete crftd - where not exists (select 1 - from cr_revisions r - where r.filename = crftd.path) + select distinct crftd.path, crftd.storage_area_key + from cr_files_to_delete crftd + where not exists (select 1 + from cr_revisions r + where r.filename = crftd.path) + + + + begin + cr_scheduled_release_exec; + end; + + + Index: openacs-4/packages/acs-content-repository/tcl/acs-content-repository-procs-postgresql.xql =================================================================== RCS file: /usr/local/cvsroot/openacs-4/packages/acs-content-repository/tcl/acs-content-repository-procs-postgresql.xql,v diff -u -N -r1.2 -r1.2.4.1 --- openacs-4/packages/acs-content-repository/tcl/acs-content-repository-procs-postgresql.xql 27 Oct 2014 16:39:08 -0000 1.2 +++ openacs-4/packages/acs-content-repository/tcl/acs-content-repository-procs-postgresql.xql 22 Feb 2021 11:47:19 -0000 1.2.4.1 @@ -13,5 +13,13 @@ + + + + select cr_scheduled_release_exec(); + + + + Index: openacs-4/packages/acs-content-repository/tcl/acs-content-repository-procs.tcl =================================================================== RCS file: /usr/local/cvsroot/openacs-4/packages/acs-content-repository/tcl/acs-content-repository-procs.tcl,v diff -u -N -r1.13 -r1.13.2.1 --- openacs-4/packages/acs-content-repository/tcl/acs-content-repository-procs.tcl 25 Jul 2018 17:44:25 -0000 1.13 +++ openacs-4/packages/acs-content-repository/tcl/acs-content-repository-procs.tcl 22 Feb 2021 11:47:19 -0000 1.13.2.1 @@ -46,8 +46,18 @@ cr_cleanup_orphaned_files } +ad_proc -private cr_cleanup_orphaned_files {} { + Helper proc to cleanup orphaned files in the content + repository. Orphaned files can be created during aborted + transactions involving the files being added to the content + repository. +} { + cr_delete_orphans [cr_get_file_creation_log] +} + + ## ## Scan AOLserver mime types and insert them into cr_mime_types ## @@ -84,7 +94,7 @@ ## -ad_proc cr_check_orphaned_files {-delete:boolean {-mtime ""}} { +ad_proc -private cr_check_orphaned_files {-delete:boolean {-mtime ""}} { Check for orphaned files in the content repository directory, and delete such files if required. Orphaned files might be created, @@ -124,6 +134,16 @@ return $result } +ad_proc -private acs_cr_scheduled_release_exec {} { + + This was handled by oracle, but since other dbs, such as PostgreSQL don't + support job submission, the job scheduling has been moved to aolserver. + (OpenACS - DanW) + +} { + + db_exec_plsql schedule_releases {} +} # # Local variables: # mode: tcl Index: openacs-4/packages/acs-content-repository/tcl/content-item-procs.tcl =================================================================== RCS file: /usr/local/cvsroot/openacs-4/packages/acs-content-repository/tcl/content-item-procs.tcl,v diff -u -N -r1.38.2.6 -r1.38.2.7 --- openacs-4/packages/acs-content-repository/tcl/content-item-procs.tcl 1 Feb 2021 11:00:34 -0000 1.38.2.6 +++ openacs-4/packages/acs-content-repository/tcl/content-item-procs.tcl 22 Feb 2021 11:47:19 -0000 1.38.2.7 @@ -172,9 +172,26 @@ @param item_id } { - return [package_exec_plsql \ - -var_list [list [list item_id $item_id]] \ - content_item del] + set result 0 + db_1row get_storage_type {select storage_type, storage_area_key from cr_items where item_id = :item_id} + + set cleanup_data [::content::revision::collect_cleanup_data \ + -item_id $item_id \ + -storage_type $storage_type] + db_transaction { + set result [package_exec_plsql \ + -var_list [list [list item_id $item_id]] \ + content_item del] + # + # In case, everything goes well in the call above, we perform + # the cleanup. + # + ::content::revision::cleanup \ + -storage_area_key $storage_area_key \ + -storage_type $storage_type \ + -data $cleanup_data + } + return $result } ad_proc -public ::content::item::rename { @@ -278,7 +295,8 @@ @param item_id item to update - @param attributes A list of pairs of additional attributes and their values to get. Each pair is a list of two elements: key => value + @param attributes A list of pairs of additional attributes and their values to get. + Each pair is a list of two elements: key => value @return @@ -1074,7 +1092,10 @@ # # # -ad_proc -public content::item::get_revision_content { -revision_id:required -item_id } { +ad_proc -public content::item::get_revision_content { + -revision_id:required + -item_id +} { Create a onerow datasource called content in the calling frame which contains all attributes for the revision (including inherited Index: openacs-4/packages/acs-content-repository/tcl/content-procs-postgresql.xql =================================================================== RCS file: /usr/local/cvsroot/openacs-4/packages/acs-content-repository/tcl/content-procs-postgresql.xql,v diff -u -N --- openacs-4/packages/acs-content-repository/tcl/content-procs-postgresql.xql 27 Oct 2014 16:39:11 -0000 1.2 +++ /dev/null 1 Jan 1970 00:00:00 -0000 @@ -1,12 +0,0 @@ - - - postgresql7.1 - - - - SELECT count(*) FROM cr_revisions WHERE substring(content, 1, 100) = substring(:name, 1, 100); - - - - - Index: openacs-4/packages/acs-content-repository/tcl/content-procs.tcl =================================================================== RCS file: /usr/local/cvsroot/openacs-4/packages/acs-content-repository/tcl/content-procs.tcl,v diff -u -N -r1.16.2.3 -r1.16.2.4 --- openacs-4/packages/acs-content-repository/tcl/content-procs.tcl 3 Jul 2020 07:27:29 -0000 1.16.2.3 +++ openacs-4/packages/acs-content-repository/tcl/content-procs.tcl 22 Feb 2021 11:47:19 -0000 1.16.2.4 @@ -77,13 +77,7 @@ file copy -force -- $client_filename $dir$content_file } - # Record an entry in the file creation log for managing orphaned - # files. - ad_mutex_eval [nsv_get mutex cr_file_creation] { - set f [open $dir/file-creation.log a] - puts $f $content_file - close $f - } + cr_add_to_file_creation_log $content_file return $content_file } @@ -93,17 +87,15 @@ Copies the string to the content repository file storage area, and it returns the relative file path from the root of the content repository file storage area. + } { ad_mutex_eval [nsv_get mutex cr_file_creation] { set content_file [cr_create_content_file_path $item_id $revision_id] set ofp [open [cr_fs_path]$content_file w] puts -nonewline $ofp $str close $ofp - - set f [open [cr_fs_path]/file-creation.log a] - puts $f $content_file - close $f } + cr_add_to_file_creation_log $content_file return $content_file } @@ -123,15 +115,18 @@ # involving file inserts in the content repository. # -ad_proc -public cr_cleanup_orphaned_files {} { +ad_proc -private cr_add_to_file_creation_log {content_file} { - Helper proc to cleanup orphaned files in the content - repository. Orphaned files can be created during aborted - transactions involving the files being added to the content - repository. + Record an entry in the file creation log for managing orphaned + files. } { - cr_delete_orphans [cr_get_file_creation_log] + set dir [cr_fs_path] + ad_mutex_eval [nsv_get mutex cr_file_creation] { + set f [open $dir/file-creation.log a] + puts $f $content_file + close $f + } } ad_proc -private cr_get_file_creation_log {} { @@ -156,19 +151,38 @@ return $content } -ad_proc -public cr_count_file_entries {name} { +ad_proc -private cr_check_file_entry {name} { - Count the number of entries from the content repository having the - specified partial path their content field. The result should be - 0 or 1 in consistent databases. + Check if an entriy from the content repository having the + specified partial path their content field exists. + @result boolean success + } { - db_string count_entries {} + db_0or1orw check_entry { + SELECT 1 FROM cr_revisions + WHERE substring(content, 1, 100) = substring(:name, 1, 100) + } } +ad_proc -private cr_count_file_entries {name} { + + Count entries an entries from the content repository having the + specified partial path their content field. + + @result integer count + +} { + db_string count_entries { + SELECT count(*) FROM cr_revisions + WHERE substring(content, 1, 100) = substring(:name, 1, 100) + } +} + + ad_proc -private cr_delete_orphans {files} { - delete orphaned files in the content repository + Delete orphaned files in the content repository. } { set dir [cr_fs_path] Index: openacs-4/packages/acs-content-repository/tcl/content-revision-procs-oracle.xql =================================================================== RCS file: /usr/local/cvsroot/openacs-4/packages/acs-content-repository/tcl/content-revision-procs-oracle.xql,v diff -u -N -r1.9 -r1.9.2.1 --- openacs-4/packages/acs-content-repository/tcl/content-revision-procs-oracle.xql 15 Aug 2018 17:00:24 -0000 1.9 +++ openacs-4/packages/acs-content-repository/tcl/content-revision-procs-oracle.xql 22 Feb 2021 11:47:19 -0000 1.9.2.1 @@ -12,7 +12,7 @@ oracle 8.1.6 - + update cr_revisions set content = empty_blob() @@ -21,6 +21,15 @@ + + + update cr_revisions + set content = empty_blob() + where revision_id = :revision_id + returning content into :1 + + + select item_id @@ -29,7 +38,7 @@ - + update cr_revisions @@ -39,16 +48,18 @@ - + + - update cr_revisions - set filename = :filename, - mime_type = :mime_type, - content_length = :tmp_size - where revision_id = :revision_id + + update cr_revisions + set mime_type = :mime_type, + content = empty_blob() + where revision_id = :revision_id + - + select storage_area_key, Index: openacs-4/packages/acs-content-repository/tcl/content-revision-procs-postgresql.xql =================================================================== RCS file: /usr/local/cvsroot/openacs-4/packages/acs-content-repository/tcl/content-revision-procs-postgresql.xql,v diff -u -N -r1.7 -r1.7.2.1 --- openacs-4/packages/acs-content-repository/tcl/content-revision-procs-postgresql.xql 15 Aug 2018 17:00:24 -0000 1.7 +++ openacs-4/packages/acs-content-repository/tcl/content-revision-procs-postgresql.xql 22 Feb 2021 11:47:19 -0000 1.7.2.1 @@ -13,24 +13,16 @@ 7.3 - + update cr_revisions set content=:content where revision_id=:revision_id - + - select item_id - from cr_revisions - where revision_id = :revision_id - - - - - update cr_revisions set mime_type = :mime_type, lob = [set __lob_id [db_string get_lob_id {select empty_lob()}]] @@ -39,7 +31,7 @@ - + update cr_revisions @@ -49,16 +41,6 @@ - - - update cr_revisions - set content = :filename, - mime_type = :mime_type, - content_length = :tmp_size - where revision_id = :revision_id - - - select storage_area_key, Index: openacs-4/packages/acs-content-repository/tcl/content-revision-procs.tcl =================================================================== RCS file: /usr/local/cvsroot/openacs-4/packages/acs-content-repository/tcl/content-revision-procs.tcl,v diff -u -N -r1.36.2.2 -r1.36.2.3 --- openacs-4/packages/acs-content-repository/tcl/content-revision-procs.tcl 3 Jul 2020 07:27:29 -0000 1.36.2.2 +++ openacs-4/packages/acs-content-repository/tcl/content-revision-procs.tcl 22 Feb 2021 11:47:19 -0000 1.36.2.3 @@ -1,6 +1,6 @@ ad_library { - Procedures to manipulate content revisions + CRUD procedures for content revisions @author Dave Bauer (dave@thedesignexperience.org) @creation-date 2004-06-04 @@ -64,7 +64,8 @@ @param is_live True is revision should be set live - @param tmp_filename file containing content to be added to revision. Caller is responsible to handle cleaning up the temporary file + @param tmp_filename file containing content to be added to revision. + The caller is responsible for cleaning up the temporary file. @param package_id @@ -133,9 +134,12 @@ -mime_type $mime_type \ -file $tmp_filename] - set query_text "insert into ${table_name}i - (revision_id, object_type, creation_user, creation_date, creation_ip, title, description, item_id, object_package_id, mime_type $attribute_names) - values (:revision_id, :content_type, :creation_user, :creation_date, :creation_ip, :title, :description, :item_id, :package_id, :mime_type $attribute_values)" + set query_text [subst { + insert into ${table_name}i + (revision_id, object_type, creation_user, creation_date, creation_ip, title, description, item_id, object_package_id, mime_type $attribute_names) + values (:revision_id, :content_type, :creation_user, :creation_date, :creation_ip, :title, :description, :item_id, :package_id, :mime_type $attribute_values) + }] + db_transaction { # An explicit lock was necessary for PostgreSQL between 8.0 and # 8.2; left the following statement here for documentary purposes @@ -148,6 +152,7 @@ # the postgres "insert into view" is rewritten by the rule into a "select" [expr {[db_driverkey ""] eq "postgresql" ? "db_0or1row" : "db_dml"}] \ insert_revision $query_text + ::content::revision::update_content \ -item_id $item_id \ -revision_id $revision_id \ @@ -162,16 +167,294 @@ return $revision_id } -ad_proc -public ::content::revision::update_content { - -item_id - -revision_id - -content - -storage_type - -mime_type - {-tmp_filename ""} +# +# ::content::revision::collect_cleanup_data +# +ad_proc -private ::content::revision::collect_cleanup_data { + -item_id:required + -storage_type:required } { + return [::content::revision::collect_cleanup_data-$storage_type -item_id $item_id] +} +ad_proc -private ::content::revision::collect_cleanup_data-text { + -item_id:required +} { + return +} + +ad_proc -private ::content::revision::collect_cleanup_data-lob { + -item_id:required +} { + return +} + +ad_proc -private ::content::revision::collect_cleanup_data-file { + -item_id:required +} { + return [db_list get_files {select content from cr_revisions where item_id = :item_id}] +} + +# +# ::content::revision::cleanup +# +ad_proc -private ::content::revision::cleanup { + -storage_type:required + -storage_area_key:required + -data:required +} { + return [::content::revision::cleanup-$storage_type \ + -storage_area_key $storage_area_key \ + -data $data] +} + +ad_proc -private ::content::revision::cleanup-text { + -storage_area_key:required + -data:required +} { + return +} + +ad_proc -private ::content::revision::cleanup-lob { + -storage_area_key:required + -data:required +} { + return +} + +ad_proc -private ::content::revision::cleanup-file { + -storage_area_key:required + -data:required +} { + + This function cleans-up files AFTER the DB-entry was deleted. If + the transaction is aborted, the file will not be executed and the + file will survive. Thus function should make + cr_check_orphaned_files obsolete, which does not scale. + + @see cr_check_orphaned_files +} { + set dir [cr_fs_path $storage_area_key] + foreach filename $data { + ns_log notice "DELETE FILE $dir$filename" + file delete $dir$filename + } +} + + +ad_proc -private ::content::revision::check_files { + {-max_results 5000} + {-max_checks 10000} + {-returnlist:boolean} +} { + Figure out, how many files in the CR are not linked to the + revisions in the content repository, and report them + optionally. + + @author Gustaf Neumann + + @param max_results stop after having found so many non-referenced files + @param max_checks stop after having checked so many non-referenced files + @param returnlist return the non-referenced files as part of the result +} { + set paths [cr_fs_path CR_FILES] + set prefix_length [string length $paths] + set count 1 + set missing 0 + set files {} + while {[llength $paths] > 0} { + # get the first path + set paths [lassign $paths path] + #ns_log notice "popping path '$path' form paths, remaining [llength $paths]" + + set children [glob -nocomplain -directory $path *] + foreach child $children { + if {[file tail $child] in {. ..}} { + continue + } + if {[file isdirectory $child]} { + # + # Using "lappend" leads to a breadth-search: might be + # slow when the directories a huge, since it takes a + # while until leaves are found. + # + #lappend paths $child + + set paths [lreplace $paths -1 -2 $child] + #ns_log notice "child is dir $child" + } else { + set suffix [string range $child $prefix_length end] + set success [cr_count_file_entries $suffix] + if {$success == 0} { + ns_log notice "check_files: $count file $child not in db entries" + incr missing + lappend files $child + } + incr count + if {$count >= $max_checks || $missing >= $max_results} break + + } + } + if {$count >= $max_checks || $missing >= $max_results} break + } + set msg "$missing of $count files are not ok (not contained in db entries)" + if {$returnlist_p} { + append msg \n [join $files \n] + } + return $msg +} + +ad_proc -private ::content::revision::check_dirs { + {-max_results 5000} + {-max_checks 10000} + {-returnlist:boolean} + {-prune:boolean} +} { + Figure out, how many directories in the CR are empty, report them + optionally or delete them optionally. + + @author Gustaf Neumann + + @param max_results stop after having found so many empty directories + @param max_checks stop after having checked so many directories + @param prune delete the found empty directories + @param returnlist return the directories as part of the result +} { + set paths [cr_fs_path CR_FILES] + set prefix_length [string length $paths] + set count 1 + set empty_dirs 0 + set dirs 0 + set empty_dir_list {} + while {[llength $paths] > 0} { + # get the first path + set paths [lassign $paths path] + #ns_log notice "popping path '$path' form paths, remaining [llength $paths]" + + set children [glob -nocomplain -directory $path *] + set nr_children 0 + incr dirs + foreach child $children { + if {[file tail $child] in {. ..}} { + continue + } + if {[file isdirectory $child]} { + # + # Using "lappend" leads to a breadth-search: might be + # slow when the directories a huge, since it takes a + # while until leaves are found. + # + #lappend paths $child + + set paths [lreplace $paths -1 -2 $child] + #ns_log notice "child is dir $child" + } + incr nr_children + } + if {$nr_children == 0} { + incr empty_dirs + ns_log notice "check_dirs: directory $path is empty ($empty_dirs out of $dirs)" + lappend empty_dir_list $path + if {$prune_p && [regexp {^\d+$} [file tail $path]]} { + file delete $path + } + } + if {$empty_dirs >= $max_results || $dirs >= $max_checks} { + break + } + } + set msg "$empty_dirs out of $dirs directories are empty" + ns_log notice "check_dirs: $msg" + if {$returnlist_p} { + append msg \n [join $empty_dir_list \n] + } + return $msg +} + +ad_proc -private ::content::revision::file_stats { + {-max 10000} +} { + + Determine some basic statistics about files in the CR based on a + sample. This is useful for large installations with several + million of files, where a detailed analysis would take very long. + + @author Gustaf Neumann + + @param max number of revisions with storage-type "file" to check + @result some statistics +} { + set tuples [db_list_of_lists get_file_names { + select i.item_id, revision_id, mime_type, content_length + from cr_items i, cr_revisions r + where storage_type = 'file' + and storage_area_key = 'CR_FILES' + and r.item_id = i.item_id + FETCH FIRST :max ROWS ONLY + }] + set count 0 + set total_length 0 + set empty_files 0 + foreach tuple $tuples { + lassign $tuple item_id revision_id mime_type content_length + incr count + if {$content_length eq ""} { + ns_log warning "file_stats: entry has no content_length: revision_id $revision_id mime_type $mime_type" + } else { + incr total_length $content_length + } + incr mime_types($mime_type) + incr revisions_for_item($item_id) + if {$content_length < 1} { + incr empty_files + } + } + set result "" + if {$count > 0} { + set backup_files 0 + set files_with_multiple_revisions 0 + foreach {item_id revs} [array get revisions_for_item] { + if {$revs > 1} { + incr files_with_multiple_revisions + incr backup_files [expr {$revs - 1}] + } + } + set most_common [lrange [lsort \ + -integer \ + -stride 2 \ + -index 1 \ + -decreasing \ + [array get mime_types] + ] 0 11] + + append result \ + "checked files : $count\n" \ + "files with multiple revisions: $files_with_multiple_revisions\n" \ + "backup files : $backup_files\n" \ + "empty files : $empty_files\n" \ + "avg file size : [format %10.2f [expr {$total_length*1.0/$count}]]\n" \ + "mime_types: $most_common" + ns_log notice "file_stats: $result" + } + return $result +} + + + + +# +# ::content::revision::update_content +# +ad_proc -private ::content::revision::update_content { + -item_id:required + -revision_id:required + -content:required + -storage_type:required + -mime_type:required + {-tmp_filename ""} +} { + Update content column separately. Oracle does not allow insert into a BLOB. @@ -187,43 +470,77 @@ @param content Content to add to resivsion @param storage_type text, file, or lob @param mime_type mime type of the content - @param tmp_filename For file storage type a filename can be specified. It will be added to the content repository. Caller is responsible to handle cleaning up the temporary file + @param tmp_filename For storage types except 'text' + a filename can be specified + instead of 'content'. The caller is responsible + for cleaning up the temporary file +} { + ns_log notice "============== update_content-$storage_type $revision_id content '$content' mime_type $mime_type tmp_filename '$tmp_filename'" + ::content::revision::update_content-$storage_type \ + -item_id $item_id \ + -revision_id $revision_id \ + -content $content \ + -mime_type $mime_type \ + -tmp_filename $tmp_filename +} - @return +ad_proc -private ::content::revision::update_content-text { + -item_id:required + -revision_id:required + -content:required + -mime_type:required + {-tmp_filename ""} +} { + db_dml update_content "" -blobs [list $content] - @error + if {$tmp_filename ne ""} { + # Traditionally, a provided tmp_file is not handled. I + # could/should be probably supported in the future. + ns_log warning "::content::revision::update_content-text: provided tmp_filename is ignored" + } +} + +ad_proc -private ::content::revision::update_content-file { + -item_id:required + -revision_id:required + -content:required + -mime_type:required + {-tmp_filename ""} } { + if {$tmp_filename eq ""} { + set filename [cr_create_content_file_from_string $item_id $revision_id $content] + } else { + set filename [cr_create_content_file $item_id $revision_id $tmp_filename] + } + set tmp_size [file size [cr_fs_path]$filename] + db_dml set_file_content { + update cr_revisions + set content = :filename, + mime_type = :mime_type, + content_length = :tmp_size + where revision_id = :revision_id + } +} - switch -- $storage_type { - file { - if {$tmp_filename eq ""} { - set filename [cr_create_content_file_from_string $item_id $revision_id $content] - } else { - set filename [cr_create_content_file $item_id $revision_id $tmp_filename] - } - set tmp_size [file size [cr_fs_path]$filename] - db_dml set_file_content "" - } - lob { - if {$tmp_filename ne ""} { - # handle file - set filename [cr_create_content_file $item_id $revision_id $tmp_filename] - db_dml set_lob_content "" -blob_files [list $tmp_filename] - db_dml set_lob_size "" - } else { - # handle blob - db_dml update_content "" -blobs [list $content] - } - } - default { - # HAM : 112505 - # I added a default switch because in some cases - # storage type is text and revision is not being updated - db_dml update_content "" -blobs [list $content] - } +ad_proc -private ::content::revision::update_content-lob { + -item_id:required + -revision_id:required + -content:required + -mime_type:required + {-tmp_filename ""} +} { + if {$tmp_filename ne ""} { + # handle file + set filename [cr_create_content_file $item_id $revision_id $tmp_filename] + db_dml set_content "" -blob_files [list $tmp_filename] + db_dml set_size "" + } else { + # handle blob + db_dml update_content "" -blobs [list $content] } } + ad_proc -public content::revision::content_copy { -revision_id:required {-revision_id_dest ""} @@ -365,7 +682,11 @@ @return The item_id of the item to which this revision belongs } { - return [db_string item_id {} -default ""] + return [db_string item_id { + select item_id + from cr_revisions + where revision_id = :revision_id + } -default ""] } @@ -493,7 +814,7 @@ # the file path is stored in filename column on oracle # and content in PostgreSQL, but we alias to filename so it makes # sense - db_1row get_storage_key_and_path "" + db_1row get_storage_key_and_path {} return [cr_fs_path $storage_area_key]${filename} } Index: openacs-4/packages/acs-content-repository/tcl/deprecated-procs.tcl =================================================================== RCS file: /usr/local/cvsroot/openacs-4/packages/acs-content-repository/tcl/deprecated-procs.tcl,v diff -u -N -r1.7.2.1 -r1.7.2.2 --- openacs-4/packages/acs-content-repository/tcl/deprecated-procs.tcl 17 Feb 2021 10:41:31 -0000 1.7.2.1 +++ openacs-4/packages/acs-content-repository/tcl/deprecated-procs.tcl 22 Feb 2021 11:47:19 -0000 1.7.2.2 @@ -44,7 +44,7 @@ set creation_ip [ad_conn peeraddr] if {$package_id eq ""} { - set package_id [ad_conn package_id] + set package_id [ad_conn package_id] } set keyword_id [db_exec_plsql content_keyword_new {}] @@ -127,19 +127,19 @@ } { # First, unassign for the parents of each/all if {$singular_p} { - foreach keyword $keyword_id { - set parent_id [db_string get_parent_id { - select parent_id - from cr_keywords - where keyword_id = :keyword - }] - item_unassign_children -item_id $item_id -parent_id $parent_id - } + foreach keyword $keyword_id { + set parent_id [db_string get_parent_id { + select parent_id + from cr_keywords + where keyword_id = :keyword + }] + item_unassign_children -item_id $item_id -parent_id $parent_id + } } # Now assign for each/all foreach keyword $keyword_id { - db_exec_plsql keyword_assign {} + db_exec_plsql keyword_assign {} } return $item_id @@ -159,9 +159,9 @@ } { if {[info exists parent_id]} { - set keyword_list [content::keyword::item_get_assigned -parent_id $parent_id -item_id $item_id] + set keyword_list [content::keyword::item_get_assigned -parent_id $parent_id -item_id $item_id] } else { - set keyword_list [content::keyword::item_get_assigned -item_id $item_id] + set keyword_list [content::keyword::item_get_assigned -item_id $item_id] } return $keyword_list @@ -221,7 +221,7 @@ set creation_ip [ad_conn peeraddr] if {$package_id eq ""} { - set package_id [ad_conn package_id] + set package_id [ad_conn package_id] } return [db_exec_plsql symlink_new {}] @@ -293,30 +293,30 @@ } ad_proc -public -deprecated content_symlink::resolve { - -item_id:required + -item_id:required } { - @param item_id item_id of content_symlink item to resolve + @param item_id item_id of content_symlink item to resolve - @return item_id of symlink target - @see content::symlink::resolve + @return item_id of symlink target + @see content::symlink::resolve } { - return [db_exec_plsql resolve_symlink ""] + return [db_exec_plsql resolve_symlink ""] } ad_proc -public -deprecated content_symlink::resolve_content_type { - -item_id:required + -item_id:required } { - @param item_id item_id of symlink + @param item_id item_id of symlink - @return content_type of target item - @see content::symlink::resolve_content_type + @return content_type of target item + @see content::symlink::resolve_content_type } { - return [db_exec_plsql resolve_content_type ""] + return [db_exec_plsql resolve_content_type ""] } @@ -615,8 +615,8 @@ set revision_id [::content::item::get_live_revision -item_id $item_id] if { [template::util::is_nil revision_id] } { - ns_log warning "item::get_best_revision: No live revision for content item $item_id" - return "" + ns_log warning "item::get_best_revision: No live revision for content item $item_id" + return "" } } else { @@ -1056,7 +1056,7 @@ set creation_ip [ad_conn peeraddr] if {$package_id eq ""} { - set package_id [ad_conn package_id] + set package_id [ad_conn package_id] } return [db_exec_plsql extlink_new {}] @@ -1184,6 +1184,90 @@ db_exec_plsql delete_folder {} } +################################################################################## +# +# From search-procs.tcl +# +################################################################################## +ad_proc -deprecated content_search__datasource { + object_id +} { + Provides data source for search interface. Used to access content items + after search. + + DEPRECATED: does not comply with OpenACS naming convention + + @see content_search::datasource +} { + return [content_search::datasource $object_id] +} + +ad_proc -deprecated content_search__url { + object_id +} { + Provides a URL for linking to content items which show up in a search + result set. + + DEPRECATED: does not comply with OpenACS naming convention + + @see content_search::url +} { + return [content_search::url $object_id] +} + +ad_proc -deprecated image_search__datasource { + object_id +} { + Provides data source for search interface. Used to access content items + after search. + + DEPRECATED: does not comply with OpenACS naming convention + + @see image_search::datasource +} { + return [image_search::datasource $object_id] +} + +ad_proc -deprecated image_search__url { + object_id +} { + Provides a URL for linking to content items which show up in a search + result set. + + DEPRECATED: does not comply with OpenACS naming convention + + @see image_search::url +} { + return [image_search::url $object_id] +} + +ad_proc -deprecated template_search__datasource { + object_id +} { + Provides data source for search interface. Used to access content items + after search. + + DEPRECATED: does not comply with OpenACS naming convention + + @see template_search::datasource +} { + return [template_search::datasource $object_id] +} + +ad_proc -deprecated template_search__url { + object_id +} { + Provides a URL for linking to content items which show up in a search + result set. + + DEPRECATED: does not comply with OpenACS naming convention + + @see template_search::url +} { + return [template_search::url $object_id] +} + + # Local variables: # mode: tcl # tcl-indent-level: 4 Index: openacs-4/packages/acs-content-repository/tcl/revision-procs-oracle.xql =================================================================== RCS file: /usr/local/cvsroot/openacs-4/packages/acs-content-repository/tcl/revision-procs-oracle.xql,v diff -u -N -r1.13.20.3 -r1.13.20.4 --- openacs-4/packages/acs-content-repository/tcl/revision-procs-oracle.xql 23 Oct 2020 17:03:05 -0000 1.13.20.3 +++ openacs-4/packages/acs-content-repository/tcl/revision-procs-oracle.xql 22 Feb 2021 11:47:19 -0000 1.13.20.4 @@ -2,7 +2,7 @@ oracle8.1.6 - + select :path || filename from cr_revisions Index: openacs-4/packages/acs-content-repository/tcl/revision-procs-postgresql.xql =================================================================== RCS file: /usr/local/cvsroot/openacs-4/packages/acs-content-repository/tcl/revision-procs-postgresql.xql,v diff -u -N -r1.11.2.3 -r1.11.2.4 --- openacs-4/packages/acs-content-repository/tcl/revision-procs-postgresql.xql 3 Jan 2020 10:46:57 -0000 1.11.2.3 +++ openacs-4/packages/acs-content-repository/tcl/revision-procs-postgresql.xql 22 Feb 2021 11:47:19 -0000 1.11.2.4 @@ -2,7 +2,7 @@ postgresql7.1 - + select :path || content from cr_revisions Index: openacs-4/packages/acs-content-repository/tcl/revision-procs.tcl =================================================================== RCS file: /usr/local/cvsroot/openacs-4/packages/acs-content-repository/tcl/revision-procs.tcl,v diff -u -N -r1.37.2.4 -r1.37.2.5 --- openacs-4/packages/acs-content-repository/tcl/revision-procs.tcl 28 Oct 2020 15:39:19 -0000 1.37.2.4 +++ openacs-4/packages/acs-content-repository/tcl/revision-procs.tcl 22 Feb 2021 11:47:19 -0000 1.37.2.5 @@ -1,15 +1,21 @@ -# upload an item revision from a file +ad_library { + Tcl API for adding file content to the database and for sending + file content to back to the client. + + @creation-date 2004-05-28 +} + ad_proc -public cr_write_content { -string:boolean -item_id -revision_id } { - Write out the specified content to the current HTML connection or return - it to the caller by using the -string flag. Only one of - item_id and revision_id should be passed to this procedure. If item_id is - provided the item's live revision will be written, otherwise the specified - revision. + Write out the specified content to the current HTTP connection or + return it to the caller by using the -string flag. Only one of + item_id and revision_id should be passed to this procedure. If + item_id is provided the item's live revision will be written, + otherwise the specified revision. This routine was written to centralize the downloading of data from the content repository. Previously, similar code was scattered among @@ -39,6 +45,7 @@ i.storage_area_key, r.mime_type, r.revision_id, + r.content, r.content_length from cr_items i, cr_revisions r where i.item_id = :item_id @@ -52,6 +59,7 @@ i.storage_area_key, r.mime_type, i.item_id, + r.content, r.content_length from cr_items i, cr_revisions r where r.revision_id = :revision_id and i.item_id = r.item_id @@ -62,107 +70,129 @@ error "Either revision_id or item_id must be specified" } - if { $storage_type ne "file" - && $storage_type ne "text" - && $storage_type ne "lob" - } { + if { [info commands ::cr_write_content-$storage_type] eq "" } { error "Storage type '$storage_type' is invalid." } - # I set content length to 0 here because otherwise I need to do - # db-specific queries for get_revision_info - if {$content_length eq ""} { - set content_length 0 - } + return [cr_write_content-$storage_type \ + -string=$string_p \ + -item_id $item_id \ + -revision_id $revision_id \ + -mime_type $mime_type \ + -content $content \ + -content_length $content_length \ + -storage_area_key $storage_area_key] +} - switch -- $storage_type { - text { - set text [db_string write_text_content { - select content - from cr_revisions - where revision_id = :revision_id - }] - if { $string_p } { - return $text - } - ns_return 200 $mime_type $text - } - file { - set path [cr_fs_path $storage_area_key] - set filename [db_string write_file_content ""] - if {$filename eq ""} { - error "No content for the revision $revision_id.\ - This seems to be an error which occurred during the upload of the file" - } elseif {![file readable $filename]} { - ns_log Error "Could not read file $filename. Maybe the content repository is (partially) missing?" - ns_return 404 text/plain {} - } else { - if { $string_p } { - set fd [open $filename "r"] - fconfigure $fd \ - -translation binary \ - -encoding [encoding system] - set text [read $fd] - close $fd - return $text - } else { - # JCD: for webdavfs there needs to be a content-length 0 header - # but ns_returnfile does not send one. Also, we need to - # ns_return size 0 files since if fastpath is enabled ns_returnfile - # simply closes the connection rather than send anything (including - # any headers). This bug is fixed in AOLServer 4.0.6 and later - # but work around it for now. - set size [file size $filename] - if {!$size} { - ns_set put [ns_conn outputheaders] "Content-Length" 0 - ns_return 200 text/plain {} - } else { - if {[namespace which ad_returnfile_background] eq "" || [security::secure_conn_p]} { - ns_returnfile 200 $mime_type $filename - } else { - ad_returnfile_background 200 $mime_type $filename - } - } - } - } - } - lob { - if { $string_p } { - return [db_blob_get write_lob_content ""] - } +ad_proc -private cr_write_content-text { + -string:boolean + -item_id + -revision_id + -mime_type + -content + -content_length + -storage_area_key +} { + if { $string_p } { + return $content + } + ns_return 200 $mime_type $content +} - # - # Need to set content_length header here. - # - # Unfortunately, old versions of OpenACS did not set the - # content_length correctly, so we fix this here locally. - # - if {$content_length eq "0" && [db_driverkey ""] eq "postgresql"} { - set content_length [db_string get_lob_length { - select sum(byte_len) - from cr_revisions, lob_data - where revision_id = :revision_id and lob_id = cr_revisions.lob - }] - } +ad_proc -private cr_write_content-file { + -string:boolean + -item_id + -revision_id + -mime_type + -content + -content_length + -storage_area_key + -filename +} { + if {![info exists filename]} { + set path [cr_fs_path $storage_area_key] + set filename $path$content + } + if {$filename eq ""} { + error "No content for the revision $revision_id.\ + This seems to be an error which occurred during the upload of the file" - ns_set put [ns_conn outputheaders] "Content-Length" $content_length + } elseif {![file readable $filename]} { + ns_log Error "Could not read file $filename. Maybe the content repository is (partially) missing?" + ns_return 404 text/plain {} - util_return_headers $mime_type $content_length - # - # In a HEAD request, just send headers and no content - # - if {![string equal -nocase "head" [ns_conn method]]} { - db_write_blob write_lob_content "" + } elseif { $string_p } { + set fd [open $filename "r"] + fconfigure $fd \ + -translation binary \ + -encoding [encoding system] + set text [read $fd] + close $fd + return $text + + } else { + # JCD: for webdavfs there needs to be a content-length 0 header + # but ns_returnfile does not send one. Also, we need to + # ns_return size 0 files since if fastpath is enabled ns_returnfile + # simply closes the connection rather than send anything (including + # any headers). This bug is fixed in AOLServer 4.0.6 and later + # but work around it for now. + set size [file size $filename] + if {$size == 0} { + ns_set put [ns_conn outputheaders] "Content-Length" 0 + ns_return 200 text/plain {} + } else { + if {[namespace which ad_returnfile_background] eq "" || [security::secure_conn_p]} { + ns_returnfile 200 $mime_type $filename } else { - ns_conn close + ad_returnfile_background 200 $mime_type $filename } } } +} - return +ad_proc -private cr_write_content-lob { + -string:boolean + -item_id + -revision_id + -mime_type + -content + -content_length + -storage_area_key +} { + + if { $string_p } { + return [db_blob_get write_lob_content ""] + } + + # + # Unfortunately, old versions of OpenACS did not set the + # content_length correctly, so we fix this here locally. + # + if {$content_length eq "" && [db_driverkey ""] eq "postgresql"} { + set content_length [db_string get_lob_length { + select sum(byte_len) + from cr_revisions, lob_data + where revision_id = :revision_id and lob_id = cr_revisions.lob + }] + } + + util_return_headers $mime_type $content_length + # + # In a HEAD request, just send headers and no content + # + if {![string equal -nocase "head" [ns_conn method]]} { + db_write_blob write_lob_content "" + } else { + ns_conn close + } } +# +# Loading content into a revision of the content repository +# + ad_proc -public cr_import_content { {-storage_type "file"} -creation_user @@ -368,25 +398,15 @@ } } - # insert the attachment into the database - - switch -- $storage_type { - file { - set filename [cr_create_content_file $item_id $revision_id $tmp_filename] - db_dml set_file_content { - update cr_revisions set - content = :filename, - mime_type = :mime_type, - content_length = :tmp_size - where revision_id = :revision_id - } - } - lob { - db_dml set_lob_content "" -blob_files [list $tmp_filename] - db_dml set_lob_size "" - } - } - + ns_log notice "TESTIONG ::content::revision::update_content -storage_type $storage_type" + # insert the content into the database + ::content::revision::update_content \ + -storage_type $storage_type \ + -item_id $item_id \ + -revision_id $revision_id \ + -content "" \ + -mime_type $mime_type \ + -tmp_filename $tmp_filename } return $revision_id @@ -555,10 +575,11 @@ @author Jeff Davis (davis@xarg.net) } { - # make both lower since that is the convention. - # should never pass in anything that is not lowercased - # already but just be safe. - + # + # Convert "mime_type" and "extension" to lowercase since that is + # the convention in the database. One should never pass in + # anything that is not lowercased already but just be safe. + # set mime_type [string tolower $mime_type] set extension [string tolower $extension] Index: openacs-4/packages/acs-content-repository/tcl/search-procs.tcl =================================================================== RCS file: /usr/local/cvsroot/openacs-4/packages/acs-content-repository/tcl/search-procs.tcl,v diff -u -N -r1.14.2.4 -r1.14.2.5 --- openacs-4/packages/acs-content-repository/tcl/search-procs.tcl 13 Dec 2019 15:02:11 -0000 1.14.2.4 +++ openacs-4/packages/acs-content-repository/tcl/search-procs.tcl 22 Feb 2021 11:47:19 -0000 1.14.2.5 @@ -5,115 +5,35 @@ contracts. } -ad_proc -deprecated content_search__datasource { - object_id -} { - Provides data source for search interface. Used to access content items - after search. - - DEPRECATED: does not comply with OpenACS naming convention - - @see content_search::datasource -} { - return [content_search::datasource $object_id] -} - -ad_proc -deprecated content_search__url { - object_id -} { - Provides a URL for linking to content items which show up in a search - result set. - - DEPRECATED: does not comply with OpenACS naming convention - - @see content_search::url -} { - return [content_search::url $object_id] -} - -ad_proc -deprecated image_search__datasource { - object_id -} { - Provides data source for search interface. Used to access content items - after search. - - DEPRECATED: does not comply with OpenACS naming convention - - @see image_search::datasource -} { - return [image_search::datasource $object_id] -} - -ad_proc -deprecated image_search__url { - object_id -} { - Provides a URL for linking to content items which show up in a search - result set. - - DEPRECATED: does not comply with OpenACS naming convention - - @see image_search::url -} { - return [image_search::url $object_id] -} - -ad_proc -deprecated template_search__datasource { - object_id -} { - Provides data source for search interface. Used to access content items - after search. - - DEPRECATED: does not comply with OpenACS naming convention - - @see template_search::datasource -} { - return [template_search::datasource $object_id] -} - -ad_proc -deprecated template_search__url { - object_id -} { - Provides a URL for linking to content items which show up in a search - result set. - - DEPRECATED: does not comply with OpenACS naming convention - - @see template_search::url -} { - return [template_search::url $object_id] -} - - namespace eval content_search {} -ad_proc content_search::datasource { +ad_proc -private content_search::datasource { object_id } { Provides data source for search interface. Used to access content items after search. } { set cr_fs_path [cr_fs_path] db_0or1row revisions_datasource { - select r.revision_id as object_id, - r.title, + select r.revision_id as object_id, + r.title, case i.storage_type when 'lob' then cast(r.lob as text) when 'file' then :cr_fs_path || r.content else r.content end as content, - r.mime_type as mime, + r.mime_type as mime, '' as keywords, - i.storage_type - from cr_revisions r, cr_items i - where revision_id = :object_id + i.storage_type + from cr_revisions r, cr_items i + where revision_id = :object_id and i.item_id = r.item_id } -column_array datasource return [array get datasource] } - -ad_proc content_search::url { +ad_proc -private content_search::url { object_id } { Provides a URL for linking to content items which show up in a search @@ -133,7 +53,7 @@ return "[ad_url][string trimright $root_url /]$url?revision_id=$object_id" } -ad_proc content_search::search_ids { +ad_proc -private content_search::search_ids { q { offset 0 } { limit 100 } @@ -151,28 +71,28 @@ namespace eval image_search {} -ad_proc image_search::datasource { +ad_proc -private image_search::datasource { object_id } { Provides data source for search interface. Used to access content items after search. } { db_0or1row revisions_datasource { - select r.revision_id as object_id, - r.title as title, - r.description as content, - r.mime_type as mime, - '' as keywords, + select r.revision_id as object_id, + r.title as title, + r.description as content, + r.mime_type as mime, + '' as keywords, 'text' as storage_type - from cr_revisions r - where revision_id = :object_id + from cr_revisions r + where revision_id = :object_id } -column_array datasource return [array get datasource] } -ad_proc image_search::url { +ad_proc -private image_search::url { object_id } { Provides a URL for linking to content items which show up in a search @@ -184,34 +104,34 @@ namespace eval template_search {} -ad_proc template_search::datasource { +ad_proc -private template_search::datasource { object_id } { Provides data source for search interface. Used to access content items after search. } { set cr_fs_path [cr_fs_path] db_0or1row revisions_datasource { - select r.revision_id as object_id, - r.title as title, + select r.revision_id as object_id, + r.title as title, case i.storage_type when 'lob' then cast(r.lob as text) when 'file' then :cr_fs_path || r.content when 'text' then r.content else r.content end as content, - r.mime_type as mime, - '' as keywords, - i.storage_type - from cr_revisions r, cr_items i - where revision_id = :object_id + r.mime_type as mime, + '' as keywords, + i.storage_type + from cr_revisions r, cr_items i + where revision_id = :object_id and i.item_id = r.item_id } -column_array datasource return [array get datasource] } -ad_proc template_search::url { +ad_proc -private template_search::url { object_id } { Provides a URL for linking to content items which show up in a search