11/19/11 18:30:21 (10 years ago)
Alaric Snell-Pym

ugarit: Only commit the file-cache sqlite database periodically, to improve performance on lots of small files.

1 edited


  • release/4/ugarit/trunk/ugarit-core.scm

    r25521 r25525  
    119119  global-directory-rules ; top-level directory rules
    120120  file-cache ; sqlite db storing filesystem cache (see store-file! procedure); #f if not enabled
     121  file-cache-get-query ; sqlite stored procedure
     122  file-cache-set-query ; sqlite stored procedure
     123  file-cache-updates-uncommitted ; count of updates since last commit
    121124  file-cache-hits ; count of file cache hits
    122125  )
    124127(include "posixextras.scm")
     129(define file-cache-commit-interval 1000)
    126131(define (file-cache-put! archive file-path mtime size key)
    127   (exec (sql (archive-file-cache archive)
    128              "INSERT OR REPLACE INTO files (path,mtime,size,key) VALUES (?,?,?,?)")
     132  (when (> file-cache-commit-interval (archive-file-cache-updates-uncommitted archive))
     133        (exec (sql (archive-file-cache archive) "commit;"))
     134        (exec (sql (archive-file-cache archive) "begin;"))
     135        (archive-file-cache-updates-uncommitted-set! archive 0))
     136  (exec (archive-file-cache-set-query archive)
    129137        file-path mtime size key))
    131139(define (file-cache-get archive file-path mtime size)
    132   (let ((data (query fetch (sql (archive-file-cache archive)
    133                                 "SELECT key FROM files WHERE path = ? AND mtime = ? AND size = ?")
     140  (let ((data (query fetch
     141                     (archive-file-cache-get-query archive)
    134142                    file-path mtime size)))
    135143    (if (pair? data)
    262270                        (change-file-mode path (bitwise-ior perm/irusr perm/iwusr))
    263271                        (when (null? (schema *file-cache*))
    264                               (exec (sql *file-cache* "CREATE TABLE files (path TEXT PRIMARY KEY, mtime INTEGER, size INTEGER, key TEXT);"))))
     272                              (exec (sql *file-cache* "CREATE TABLE files (path TEXT PRIMARY KEY, mtime INTEGER, size INTEGER, key TEXT);")))
     273                        (exec (sql *file-cache* "begin;")))
    265274                       (('rule . conf) (set! *global-rules* (cons conf *global-rules*)))
    266275                       (_ (signal (make-property-condition 'exn 'location 'open-archive 'message "Unknown configuration entry" 'arguments (list confentry))))))
    286295       decrypt
    287296       *global-rules*
    288        *file-cache* 0))))
     297       *file-cache*
     298       (if *file-cache* (sql *file-cache* "SELECT key FROM files WHERE path = ? AND mtime = ? AND size = ?") #f)
     299       (if *file-cache* (sql *file-cache* "INSERT OR REPLACE INTO files (path,mtime,size,key) VALUES (?,?,?,?)") #f)
     300       0 0))))
    290302                                        ; Take a block, and return a compressed and encrypted block
    375387(define (archive-close! archive)
    376388  (when (archive-file-cache archive)
     389        (exec (sql (archive-file-cache archive) "commit;"))
    377390        (close-database (archive-file-cache archive)))
    378391  ((storage-close! (archive-storage archive))))
    551564;; Uses standard input port for the file data
    552565;; Returns key and reused?
     566;; FIXME: Needs much refactoring, the file cache was hacked in willy-nilly.
    553567(define (store-file! archive file-path file-stat)
    554568  (let* ((store-file-without-caching!
    583597               (size (vector-ref file-stat 5))
    584598               (cache-result (file-cache-get archive file-path mtime size)))
    585           (if cache-result
     599          (if cache-result ;; FIXME: This assumes that the cached file IS in the archive. Give a configurable option to make it check this, making the file-cache a file hash cache rather than also being an archive presence cache like backend-cache as well, for safety.
    586600              (begin
    587601                (archive-file-cache-hits-set! archive
Note: See TracChangeset for help on using the changeset viewer.