diff options
Diffstat (limited to 'subversion/libsvn_fs_x')
51 files changed, 7174 insertions, 6227 deletions
diff --git a/subversion/libsvn_fs_x/batch_fsync.c b/subversion/libsvn_fs_x/batch_fsync.c new file mode 100644 index 000000000000..4116d7d7614c --- /dev/null +++ b/subversion/libsvn_fs_x/batch_fsync.c @@ -0,0 +1,588 @@ +/* batch_fsync.c --- efficiently fsync multiple targets + * + * ==================================================================== + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * ==================================================================== + */ + +#include <apr_thread_pool.h> +#include <apr_thread_cond.h> + +#include "batch_fsync.h" +#include "svn_pools.h" +#include "svn_hash.h" +#include "svn_dirent_uri.h" +#include "svn_private_config.h" + +#include "private/svn_atomic.h" +#include "private/svn_dep_compat.h" +#include "private/svn_mutex.h" +#include "private/svn_subr_private.h" + +/* Handy macro to check APR function results and turning them into + * svn_error_t upon failure. */ +#define WRAP_APR_ERR(x,msg) \ + { \ + apr_status_t status_ = (x); \ + if (status_) \ + return svn_error_wrap_apr(status_, msg); \ + } + + +/* A simple SVN-wrapper around the apr_thread_cond_* API */ +#if APR_HAS_THREADS +typedef apr_thread_cond_t svn_thread_cond__t; +#else +typedef int svn_thread_cond__t; +#endif + +static svn_error_t * +svn_thread_cond__create(svn_thread_cond__t **cond, + apr_pool_t *result_pool) +{ +#if APR_HAS_THREADS + + WRAP_APR_ERR(apr_thread_cond_create(cond, result_pool), + _("Can't create condition variable")); + +#else + + *cond = apr_pcalloc(result_pool, sizeof(**cond)); + +#endif + + return SVN_NO_ERROR; +} + +static svn_error_t * +svn_thread_cond__broadcast(svn_thread_cond__t *cond) +{ +#if APR_HAS_THREADS + + WRAP_APR_ERR(apr_thread_cond_broadcast(cond), + _("Can't broadcast condition variable")); + +#endif + + return SVN_NO_ERROR; +} + +static svn_error_t * +svn_thread_cond__wait(svn_thread_cond__t *cond, + svn_mutex__t *mutex) +{ +#if APR_HAS_THREADS + + WRAP_APR_ERR(apr_thread_cond_wait(cond, svn_mutex__get(mutex)), + _("Can't broadcast condition variable")); + +#endif + + return SVN_NO_ERROR; +} + +/* Utility construct: Clients can efficiently wait for the encapsulated + * counter to reach a certain value. Currently, only increments have been + * implemented. This whole structure can be opaque to the API users. + */ +typedef struct waitable_counter_t +{ + /* Current value, initialized to 0. */ + int value; + + /* Synchronization objects. */ + svn_thread_cond__t *cond; + svn_mutex__t *mutex; +} waitable_counter_t; + +/* Set *COUNTER_P to a new waitable_counter_t instance allocated in + * RESULT_POOL. The initial counter value is 0. */ +static svn_error_t * +waitable_counter__create(waitable_counter_t **counter_p, + apr_pool_t *result_pool) +{ + waitable_counter_t *counter = apr_pcalloc(result_pool, sizeof(*counter)); + counter->value = 0; + + SVN_ERR(svn_thread_cond__create(&counter->cond, result_pool)); + SVN_ERR(svn_mutex__init(&counter->mutex, TRUE, result_pool)); + + *counter_p = counter; + + return SVN_NO_ERROR; +} + +/* Increment the value in COUNTER by 1. */ +static svn_error_t * +waitable_counter__increment(waitable_counter_t *counter) +{ + SVN_ERR(svn_mutex__lock(counter->mutex)); + counter->value++; + + SVN_ERR(svn_thread_cond__broadcast(counter->cond)); + SVN_ERR(svn_mutex__unlock(counter->mutex, SVN_NO_ERROR)); + + return SVN_NO_ERROR; +} + +/* Efficiently wait for COUNTER to assume VALUE. */ +static svn_error_t * +waitable_counter__wait_for(waitable_counter_t *counter, + int value) +{ + svn_boolean_t done = FALSE; + + /* This loop implicitly handles spurious wake-ups. */ + do + { + SVN_ERR(svn_mutex__lock(counter->mutex)); + + if (counter->value == value) + done = TRUE; + else + SVN_ERR(svn_thread_cond__wait(counter->cond, counter->mutex)); + + SVN_ERR(svn_mutex__unlock(counter->mutex, SVN_NO_ERROR)); + } + while (!done); + + return SVN_NO_ERROR; +} + +/* Set the value in COUNTER to 0. */ +static svn_error_t * +waitable_counter__reset(waitable_counter_t *counter) +{ + SVN_ERR(svn_mutex__lock(counter->mutex)); + counter->value = 0; + SVN_ERR(svn_mutex__unlock(counter->mutex, SVN_NO_ERROR)); + + SVN_ERR(svn_thread_cond__broadcast(counter->cond)); + + return SVN_NO_ERROR; +} + +/* Entry type for the svn_fs_x__batch_fsync_t collection. There is one + * instance per file handle. + */ +typedef struct to_sync_t +{ + /* Open handle of the file / directory to fsync. */ + apr_file_t *file; + + /* Pool to use with FILE. It is private to FILE such that it can be + * used safely together with FILE in a separate thread. */ + apr_pool_t *pool; + + /* Result of the file operations. */ + svn_error_t *result; + + /* Counter to increment when we completed the task. */ + waitable_counter_t *counter; +} to_sync_t; + +/* The actual collection object. */ +struct svn_fs_x__batch_fsync_t +{ + /* Maps open file handles: C-string path to to_sync_t *. */ + apr_hash_t *files; + + /* Counts the number of completed fsync tasks. */ + waitable_counter_t *counter; + + /* Perform fsyncs only if this flag has been set. */ + svn_boolean_t flush_to_disk; +}; + +/* Data structures for concurrent fsync execution are only available if + * we have threading support. + */ +#if APR_HAS_THREADS + +/* Number of microseconds that an unused thread remains in the pool before + * being terminated. + * + * Higher values are useful if clients frequently send small requests and + * you want to minimize the latency for those. + */ +#define THREADPOOL_THREAD_IDLE_LIMIT 1000000 + +/* Maximum number of threads in THREAD_POOL, i.e. number of paths we can + * fsync concurrently throughout the process. */ +#define MAX_THREADS 16 + +/* Thread pool to execute the fsync tasks. */ +static apr_thread_pool_t *thread_pool = NULL; + +#endif + +/* Keep track on whether we already created the THREAD_POOL . */ +static svn_atomic_t thread_pool_initialized = FALSE; + +/* We open non-directory files with these flags. */ +#define FILE_FLAGS (APR_READ | APR_WRITE | APR_BUFFERED | APR_CREATE) + +#if APR_HAS_THREADS + +/* Destructor function that implicitly cleans up any running threads + in the TRHEAD_POOL *once*. + + Must be run as a pre-cleanup hook. + */ +static apr_status_t +thread_pool_pre_cleanup(void *data) +{ + apr_thread_pool_t *tp = thread_pool; + if (!thread_pool) + return APR_SUCCESS; + + thread_pool = NULL; + thread_pool_initialized = FALSE; + + return apr_thread_pool_destroy(tp); +} + +#endif + +/* Core implementation of svn_fs_x__batch_fsync_init. */ +static svn_error_t * +create_thread_pool(void *baton, + apr_pool_t *owning_pool) +{ +#if APR_HAS_THREADS + /* The thread-pool must be allocated from a thread-safe pool. + GLOBAL_POOL may be single-threaded, though. */ + apr_pool_t *pool = svn_pool_create(NULL); + + /* This thread pool will get cleaned up automatically when GLOBAL_POOL + gets cleared. No additional cleanup callback is needed. */ + WRAP_APR_ERR(apr_thread_pool_create(&thread_pool, 0, MAX_THREADS, pool), + _("Can't create fsync thread pool in FSX")); + + /* Work around an APR bug: The cleanup must happen in the pre-cleanup + hook instead of the normal cleanup hook. Otherwise, the sub-pools + containing the thread objects would already be invalid. */ + apr_pool_pre_cleanup_register(pool, NULL, thread_pool_pre_cleanup); + apr_pool_pre_cleanup_register(owning_pool, NULL, thread_pool_pre_cleanup); + + /* let idle threads linger for a while in case more requests are + coming in */ + apr_thread_pool_idle_wait_set(thread_pool, THREADPOOL_THREAD_IDLE_LIMIT); + + /* don't queue requests unless we reached the worker thread limit */ + apr_thread_pool_threshold_set(thread_pool, 0); + +#endif + + return SVN_NO_ERROR; +} + +svn_error_t * +svn_fs_x__batch_fsync_init(apr_pool_t *owning_pool) +{ + /* Protect against multiple calls. */ + return svn_error_trace(svn_atomic__init_once(&thread_pool_initialized, + create_thread_pool, + NULL, owning_pool)); +} + +/* Destructor for svn_fs_x__batch_fsync_t. Releases all global pool memory + * and closes all open file handles. */ +static apr_status_t +fsync_batch_cleanup(void *data) +{ + svn_fs_x__batch_fsync_t *batch = data; + apr_hash_index_t *hi; + + /* Close all files (implicitly) and release memory. */ + for (hi = apr_hash_first(apr_hash_pool_get(batch->files), batch->files); + hi; + hi = apr_hash_next(hi)) + { + to_sync_t *to_sync = apr_hash_this_val(hi); + svn_pool_destroy(to_sync->pool); + } + + return APR_SUCCESS; +} + +svn_error_t * +svn_fs_x__batch_fsync_create(svn_fs_x__batch_fsync_t **result_p, + svn_boolean_t flush_to_disk, + apr_pool_t *result_pool) +{ + svn_fs_x__batch_fsync_t *result = apr_pcalloc(result_pool, sizeof(*result)); + result->files = svn_hash__make(result_pool); + result->flush_to_disk = flush_to_disk; + + SVN_ERR(waitable_counter__create(&result->counter, result_pool)); + apr_pool_cleanup_register(result_pool, result, fsync_batch_cleanup, + apr_pool_cleanup_null); + + *result_p = result; + + return SVN_NO_ERROR; +} + +/* If BATCH does not contain a handle for PATH, yet, create one with FLAGS + * and add it to BATCH. Set *FILE to the open file handle. + * Use SCRATCH_POOL for temporaries. + */ +static svn_error_t * +internal_open_file(apr_file_t **file, + svn_fs_x__batch_fsync_t *batch, + const char *path, + apr_int32_t flags, + apr_pool_t *scratch_pool) +{ + svn_error_t *err; + apr_pool_t *pool; + to_sync_t *to_sync; +#ifdef SVN_ON_POSIX + svn_boolean_t is_new_file; +#endif + + /* If we already have a handle for PATH, return that. */ + to_sync = svn_hash_gets(batch->files, path); + if (to_sync) + { + *file = to_sync->file; + return SVN_NO_ERROR; + } + + /* Calling fsync in PATH is going to be expensive in any case, so we can + * allow for some extra overhead figuring out whether the file already + * exists. If it doesn't, be sure to schedule parent folder updates, if + * required on this platform. + * + * See svn_fs_x__batch_fsync_new_path() for when such extra fsyncs may be + * needed at all. */ + +#ifdef SVN_ON_POSIX + + is_new_file = FALSE; + if (flags & APR_CREATE) + { + svn_node_kind_t kind; + /* We might actually be about to create a new file. + * Check whether the file already exists. */ + SVN_ERR(svn_io_check_path(path, &kind, scratch_pool)); + is_new_file = kind == svn_node_none; + } + +#endif + + /* To be able to process each file in a separate thread, they must use + * separate, thread-safe pools. Allocating a sub-pool from the standard + * memory pool achieves exactly that. */ + pool = svn_pool_create(NULL); + err = svn_io_file_open(file, path, flags, APR_OS_DEFAULT, pool); + if (err) + { + svn_pool_destroy(pool); + return svn_error_trace(err); + } + + to_sync = apr_pcalloc(pool, sizeof(*to_sync)); + to_sync->file = *file; + to_sync->pool = pool; + to_sync->result = SVN_NO_ERROR; + to_sync->counter = batch->counter; + + svn_hash_sets(batch->files, + apr_pstrdup(apr_hash_pool_get(batch->files), path), + to_sync); + + /* If we just created a new file, schedule any additional necessary fsyncs. + * Note that this can only recurse once since the parent folder already + * exists on disk. */ +#ifdef SVN_ON_POSIX + + if (is_new_file) + SVN_ERR(svn_fs_x__batch_fsync_new_path(batch, path, scratch_pool)); + +#endif + + return SVN_NO_ERROR; +} + +svn_error_t * +svn_fs_x__batch_fsync_open_file(apr_file_t **file, + svn_fs_x__batch_fsync_t *batch, + const char *filename, + apr_pool_t *scratch_pool) +{ + apr_off_t offset = 0; + + SVN_ERR(internal_open_file(file, batch, filename, FILE_FLAGS, + scratch_pool)); + SVN_ERR(svn_io_file_seek(*file, APR_SET, &offset, scratch_pool)); + + return SVN_NO_ERROR; +} + +svn_error_t * +svn_fs_x__batch_fsync_new_path(svn_fs_x__batch_fsync_t *batch, + const char *path, + apr_pool_t *scratch_pool) +{ + apr_file_t *file; + +#ifdef SVN_ON_POSIX + + /* On POSIX, we need to sync the parent directory because it contains + * the name for the file / folder given by PATH. */ + path = svn_dirent_dirname(path, scratch_pool); + SVN_ERR(internal_open_file(&file, batch, path, APR_READ, scratch_pool)); + +#else + + svn_node_kind_t kind; + + /* On non-POSIX systems, we assume that sync'ing the given PATH is the + * right thing to do. Also, we assume that only files may be sync'ed. */ + SVN_ERR(svn_io_check_path(path, &kind, scratch_pool)); + if (kind == svn_node_file) + SVN_ERR(internal_open_file(&file, batch, path, FILE_FLAGS, + scratch_pool)); + +#endif + + return SVN_NO_ERROR; +} + +/* Thread-pool task Flush the to_sync_t instance given by DATA. */ +static void * APR_THREAD_FUNC +flush_task(apr_thread_t *tid, + void *data) +{ + to_sync_t *to_sync = data; + + to_sync->result = svn_error_trace(svn_io_file_flush_to_disk + (to_sync->file, to_sync->pool)); + + /* As soon as the increment call returns, TO_SYNC may be invalid + (the main thread may have woken up and released the struct. + + Therefore, we cannot chain this error into TO_SYNC->RESULT. + OTOH, the main thread will probably deadlock anyway if we got + an error here, thus there is no point in trying to tell the + main thread what the problem was. */ + svn_error_clear(waitable_counter__increment(to_sync->counter)); + + return NULL; +} + +svn_error_t * +svn_fs_x__batch_fsync_run(svn_fs_x__batch_fsync_t *batch, + apr_pool_t *scratch_pool) +{ + apr_hash_index_t *hi; + + /* Number of tasks sent to the thread pool. */ + int tasks = 0; + + /* Because we allocated the open files from our global pool, don't bail + * out on the first error. Instead, process all files and but accumulate + * the errors in this chain. + */ + svn_error_t *chain = SVN_NO_ERROR; + + /* First, flush APR-internal buffers. This should minimize / prevent the + * introduction of additional meta-data changes during the next phase. + * We might otherwise issue redundant fsyncs. + */ + for (hi = apr_hash_first(scratch_pool, batch->files); + hi; + hi = apr_hash_next(hi)) + { + to_sync_t *to_sync = apr_hash_this_val(hi); + to_sync->result = svn_error_trace(svn_io_file_flush + (to_sync->file, to_sync->pool)); + } + + /* Make sure the task completion counter is set to 0. */ + chain = svn_error_compose_create(chain, + waitable_counter__reset(batch->counter)); + + /* Start the actual fsyncing process. */ + if (batch->flush_to_disk) + { + for (hi = apr_hash_first(scratch_pool, batch->files); + hi; + hi = apr_hash_next(hi)) + { + to_sync_t *to_sync = apr_hash_this_val(hi); + +#if APR_HAS_THREADS + + /* Forgot to call _init() or cleaned up the owning pool too early? + */ + SVN_ERR_ASSERT(thread_pool); + + /* If there are multiple fsyncs to perform, run them in parallel. + * Otherwise, skip the thread-pool and synchronization overhead. */ + if (apr_hash_count(batch->files) > 1) + { + apr_status_t status = APR_SUCCESS; + status = apr_thread_pool_push(thread_pool, flush_task, to_sync, + 0, NULL); + if (status) + to_sync->result = svn_error_wrap_apr(status, + _("Can't push task")); + else + tasks++; + } + else + +#endif + + { + to_sync->result = svn_error_trace(svn_io_file_flush_to_disk + (to_sync->file, + to_sync->pool)); + } + } + } + + /* Wait for all outstanding flush operations to complete. */ + chain = svn_error_compose_create(chain, + waitable_counter__wait_for(batch->counter, + tasks)); + + /* Collect the results, close all files and release memory. */ + for (hi = apr_hash_first(scratch_pool, batch->files); + hi; + hi = apr_hash_next(hi)) + { + to_sync_t *to_sync = apr_hash_this_val(hi); + if (batch->flush_to_disk) + chain = svn_error_compose_create(chain, to_sync->result); + + chain = svn_error_compose_create(chain, + svn_io_file_close(to_sync->file, + scratch_pool)); + svn_pool_destroy(to_sync->pool); + } + + /* Don't process any file / folder twice. */ + apr_hash_clear(batch->files); + + /* Report the errors that we encountered. */ + return svn_error_trace(chain); +} diff --git a/subversion/libsvn_fs_x/batch_fsync.h b/subversion/libsvn_fs_x/batch_fsync.h new file mode 100644 index 000000000000..e3c6a289ff0d --- /dev/null +++ b/subversion/libsvn_fs_x/batch_fsync.h @@ -0,0 +1,92 @@ +/* batch_fsync.h --- efficiently fsync multiple targets + * + * ==================================================================== + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * ==================================================================== + */ + +#ifndef SVN_LIBSVN_FS_X__BATCH_FSYNC_H +#define SVN_LIBSVN_FS_X__BATCH_FSYNC_H + +#include "svn_error.h" + +/* Infrastructure for efficiently calling fsync on files and directories. + * + * The idea is to have a container of open file handles (including + * directory handles on POSIX), at most one per file. During the course + * of an FS operation that needs to be fsync'ed, all touched files and + * folders accumulate in the container. + * + * At the end of the FS operation, all file changes will be written the + * physical disk, once per file and folder. Afterwards, all handles will + * be closed and the container is ready for reuse. + * + * To minimize the delay caused by the batch flush, run all fsync calls + * concurrently - if the OS supports multi-threading. + */ + +/* Opaque container type. + */ +typedef struct svn_fs_x__batch_fsync_t svn_fs_x__batch_fsync_t; + +/* Initialize the concurrent fsync infrastructure. Clean it up when + * OWNING_POOL gets cleared. + * + * This function must be called before using any of the other functions in + * in this module. It should only be called once. + */ +svn_error_t * +svn_fs_x__batch_fsync_init(apr_pool_t *owning_pool); + +/* Set *RESULT_P to a new batch fsync structure, allocated in RESULT_POOL. + * If FLUSH_TO_DISK is not set, the resulting struct will not actually use + * fsync. */ +svn_error_t * +svn_fs_x__batch_fsync_create(svn_fs_x__batch_fsync_t **result_p, + svn_boolean_t flush_to_disk, + apr_pool_t *result_pool); + +/* Open the file at FILENAME for read and write access. Return it in *FILE + * and schedule it for fsync in BATCH. If BATCH already contains an open + * file for FILENAME, return that instead creating a new instance. + * + * Use SCRATCH_POOL for temporaries. */ +svn_error_t * +svn_fs_x__batch_fsync_open_file(apr_file_t **file, + svn_fs_x__batch_fsync_t *batch, + const char *filename, + apr_pool_t *scratch_pool); + +/* Inform the BATCH that a file or directory has been created at PATH. + * "Created" means either newly created to renamed to PATH - even if another + * item with the same name existed before. Depending on the OS, the correct + * path will scheduled for fsync. + * + * Use SCRATCH_POOL for temporaries. */ +svn_error_t * +svn_fs_x__batch_fsync_new_path(svn_fs_x__batch_fsync_t *batch, + const char *path, + apr_pool_t *scratch_pool); + +/* For all files and directories in BATCH, flush all changes to disk and + * close the file handles. Use SCRATCH_POOL for temporaries. */ +svn_error_t * +svn_fs_x__batch_fsync_run(svn_fs_x__batch_fsync_t *batch, + apr_pool_t *scratch_pool); + +#endif diff --git a/subversion/libsvn_fs_x/cached_data.c b/subversion/libsvn_fs_x/cached_data.c index 2fdf5699962b..909842e5c456 100644 --- a/subversion/libsvn_fs_x/cached_data.c +++ b/subversion/libsvn_fs_x/cached_data.c @@ -30,6 +30,7 @@ #include "private/svn_io_private.h" #include "private/svn_sorts_private.h" +#include "private/svn_string_private.h" #include "private/svn_subr_private.h" #include "private/svn_temp_serializer.h" @@ -54,6 +55,7 @@ block_read(void **result, svn_fs_t *fs, const svn_fs_x__id_t *id, svn_fs_x__revision_file_t *revision_file, + void *baton, apr_pool_t *result_pool, apr_pool_t *scratch_pool); @@ -67,7 +69,7 @@ block_read(void **result, * contents if not NULL. Use SCRATCH_POOL for temporary allocations. */ static svn_error_t * -dgb__log_access(svn_fs_t *fs, +dbg__log_access(svn_fs_t *fs, const svn_fs_x__id_t *id, void *item, apr_uint32_t item_type, @@ -173,21 +175,6 @@ dgb__log_access(svn_fs_t *fs, return SVN_NO_ERROR; } -/* Convenience wrapper around svn_io_file_aligned_seek, taking filesystem - FS instead of a block size. */ -static svn_error_t * -aligned_seek(svn_fs_t *fs, - apr_file_t *file, - apr_off_t *buffer_start, - apr_off_t offset, - apr_pool_t *scratch_pool) -{ - svn_fs_x__data_t *ffd = fs->fsap_data; - return svn_error_trace(svn_io_file_aligned_seek(file, ffd->block_size, - buffer_start, offset, - scratch_pool)); -} - /* Open the revision file for the item given by ID in filesystem FS and store the newly opened file in FILE. Seek to the item's location before returning. @@ -207,11 +194,10 @@ open_and_seek_revision(svn_fs_x__revision_file_t **file, SVN_ERR(svn_fs_x__ensure_revision_exists(rev, fs, scratch_pool)); - SVN_ERR(svn_fs_x__open_pack_or_rev_file(&rev_file, fs, rev, result_pool, - scratch_pool)); + SVN_ERR(svn_fs_x__rev_file_init(&rev_file, fs, rev, result_pool)); SVN_ERR(svn_fs_x__item_offset(&offset, &sub_item, fs, rev_file, id, scratch_pool)); - SVN_ERR(aligned_seek(fs, rev_file->file, NULL, offset, scratch_pool)); + SVN_ERR(svn_fs_x__rev_file_seek(rev_file, NULL, offset)); *file = rev_file; @@ -233,12 +219,12 @@ open_and_seek_transaction(svn_fs_x__revision_file_t **file, apr_uint32_t sub_item = 0; apr_int64_t txn_id = svn_fs_x__get_txn_id(rep->id.change_set); - SVN_ERR(svn_fs_x__open_proto_rev_file(file, fs, txn_id, result_pool, - scratch_pool)); + SVN_ERR(svn_fs_x__rev_file_open_proto_rev(file, fs, txn_id, result_pool, + scratch_pool)); SVN_ERR(svn_fs_x__item_offset(&offset, &sub_item, fs, *file, &rep->id, scratch_pool)); - SVN_ERR(aligned_seek(fs, (*file)->file, NULL, offset, scratch_pool)); + SVN_ERR(svn_fs_x__rev_file_seek(*file, NULL, offset)); return SVN_NO_ERROR; } @@ -294,6 +280,7 @@ get_node_revision_body(svn_fs_x__noderev_t **noderev_p, if (svn_fs_x__is_txn(id->change_set)) { apr_file_t *file; + svn_stream_t *stream; /* This is a transaction node-rev. Its storage logic is very different from that of rev / pack files. */ @@ -303,21 +290,19 @@ get_node_revision_body(svn_fs_x__noderev_t **noderev_p, scratch_pool), APR_READ | APR_BUFFERED, APR_OS_DEFAULT, scratch_pool); - if (err) + if (err && APR_STATUS_IS_ENOENT(err->apr_err)) + { + svn_error_clear(err); + return svn_error_trace(err_dangling_id(fs, id)); + } + else if (err) { - if (APR_STATUS_IS_ENOENT(err->apr_err)) - { - svn_error_clear(err); - return svn_error_trace(err_dangling_id(fs, id)); - } - return svn_error_trace(err); } - SVN_ERR(svn_fs_x__read_noderev(noderev_p, - svn_stream_from_aprfile2(file, - FALSE, - scratch_pool), + /* Be sure to close the file ASAP. */ + stream = svn_stream_from_aprfile2(file, FALSE, scratch_pool); + SVN_ERR(svn_fs_x__read_noderev(noderev_p, stream, result_pool, scratch_pool)); } else @@ -328,8 +313,8 @@ get_node_revision_body(svn_fs_x__noderev_t **noderev_p, svn_revnum_t revision = svn_fs_x__get_revnum(id->change_set); svn_fs_x__pair_cache_key_t key; - SVN_ERR(svn_fs_x__open_pack_or_rev_file(&revision_file, fs, revision, - scratch_pool, scratch_pool)); + SVN_ERR(svn_fs_x__rev_file_init(&revision_file, fs, revision, + scratch_pool)); /* First, try a noderevs container cache lookup. */ if ( svn_fs_x__is_packed_rev(fs, revision) @@ -355,22 +340,20 @@ get_node_revision_body(svn_fs_x__noderev_t **noderev_p, /* Not found or not applicable. Try a noderev cache lookup. * If that succeeds, we are done here. */ - if (ffd->node_revision_cache) - { - SVN_ERR(svn_cache__get((void **) noderev_p, - &is_cached, - ffd->node_revision_cache, - &key, - result_pool)); - if (is_cached) - return SVN_NO_ERROR; - } + SVN_ERR(svn_cache__get((void **) noderev_p, + &is_cached, + ffd->node_revision_cache, + &key, + result_pool)); + if (is_cached) + return SVN_NO_ERROR; /* block-read will parse the whole block and will also return the one noderev that we need right now. */ SVN_ERR(block_read((void **)noderev_p, fs, id, revision_file, + NULL, result_pool, scratch_pool)); SVN_ERR(svn_fs_x__close_revision_file(revision_file)); @@ -396,7 +379,7 @@ svn_fs_x__get_node_revision(svn_fs_x__noderev_t **noderev_p, id_string->data); } - SVN_ERR(dgb__log_access(fs, id, *noderev_p, + SVN_ERR(dbg__log_access(fs, id, *noderev_p, SVN_FS_X__ITEM_TYPE_NODEREV, scratch_pool)); return svn_error_trace(err); @@ -423,8 +406,8 @@ svn_fs_x__get_mergeinfo_count(apr_int64_t *count, svn_revnum_t revision = svn_fs_x__get_revnum(id->change_set); svn_fs_x__revision_file_t *rev_file; - SVN_ERR(svn_fs_x__open_pack_or_rev_file(&rev_file, fs, revision, - scratch_pool, scratch_pool)); + SVN_ERR(svn_fs_x__rev_file_init(&rev_file, fs, revision, + scratch_pool)); if ( svn_fs_x__is_packed_rev(fs, revision) && ffd->noderevs_container_cache) @@ -505,39 +488,13 @@ typedef struct rep_state_t int chunk_index; /* number of the window to read */ } rep_state_t; -/* Simple wrapper around svn_fs_x__get_file_offset to simplify callers. */ -static svn_error_t * -get_file_offset(apr_off_t *offset, - rep_state_t *rs, - apr_pool_t *scratch_pool) -{ - return svn_error_trace(svn_fs_x__get_file_offset(offset, - rs->sfile->rfile->file, - scratch_pool)); -} - -/* Simple wrapper around svn_io_file_aligned_seek to simplify callers. */ -static svn_error_t * -rs_aligned_seek(rep_state_t *rs, - apr_off_t *buffer_start, - apr_off_t offset, - apr_pool_t *scratch_pool) -{ - svn_fs_x__data_t *ffd = rs->sfile->fs->fsap_data; - return svn_error_trace(svn_io_file_aligned_seek(rs->sfile->rfile->file, - ffd->block_size, - buffer_start, offset, - scratch_pool)); -} - /* Open FILE->FILE and FILE->STREAM if they haven't been opened, yet. */ static svn_error_t* auto_open_shared_file(shared_file_t *file) { if (file->rfile == NULL) - SVN_ERR(svn_fs_x__open_pack_or_rev_file(&file->rfile, file->fs, - file->revision, file->pool, - file->pool)); + SVN_ERR(svn_fs_x__rev_file_init(&file->rfile, file->fs, + file->revision, file->pool)); return SVN_NO_ERROR; } @@ -571,9 +528,8 @@ auto_read_diff_version(rep_state_t *rs, if (rs->ver == -1) { char buf[4]; - SVN_ERR(rs_aligned_seek(rs, NULL, rs->start, scratch_pool)); - SVN_ERR(svn_io_file_read_full2(rs->sfile->rfile->file, buf, - sizeof(buf), NULL, NULL, scratch_pool)); + SVN_ERR(svn_fs_x__rev_file_seek(rs->sfile->rfile, NULL, rs->start)); + SVN_ERR(svn_fs_x__rev_file_read(rs->sfile->rfile, buf, sizeof(buf))); /* ### Layering violation */ if (! ((buf[0] == 'S') && (buf[1] == 'V') && (buf[2] == 'N'))) @@ -655,7 +611,7 @@ create_rep_state_body(rep_state_t **rep_state, : NULL; /* cache lookup, i.e. skip reading the rep header if possible */ - if (ffd->rep_header_cache && SVN_IS_VALID_REVNUM(revision)) + if (SVN_IS_VALID_REVNUM(revision)) SVN_ERR(svn_cache__get((void **) &rh, &is_cached, ffd->rep_header_cache, &key, result_pool)); @@ -680,6 +636,8 @@ create_rep_state_body(rep_state_t **rep_state, /* read rep header, if necessary */ if (!is_cached) { + svn_stream_t *stream; + /* we will need the on-disk location for non-txn reps */ apr_off_t offset; svn_boolean_t in_container = TRUE; @@ -731,26 +689,26 @@ create_rep_state_body(rep_state_t **rep_state, return SVN_NO_ERROR; } - SVN_ERR(rs_aligned_seek(rs, NULL, offset, scratch_pool)); + SVN_ERR(svn_fs_x__rev_file_seek(rs->sfile->rfile, NULL, offset)); } - SVN_ERR(svn_fs_x__read_rep_header(&rh, rs->sfile->rfile->stream, + SVN_ERR(svn_fs_x__rev_file_stream(&stream, rs->sfile->rfile)); + SVN_ERR(svn_fs_x__read_rep_header(&rh, stream, result_pool, scratch_pool)); - SVN_ERR(get_file_offset(&rs->start, rs, result_pool)); + SVN_ERR(svn_fs_x__rev_file_offset(&rs->start, rs->sfile->rfile)); /* populate the cache if appropriate */ if (SVN_IS_VALID_REVNUM(revision)) { - SVN_ERR(block_read(NULL, fs, &rs->rep_id, rs->sfile->rfile, + SVN_ERR(block_read(NULL, fs, &rs->rep_id, rs->sfile->rfile, NULL, result_pool, scratch_pool)); - if (ffd->rep_header_cache) - SVN_ERR(svn_cache__set(ffd->rep_header_cache, &key, rh, - scratch_pool)); + SVN_ERR(svn_cache__set(ffd->rep_header_cache, &key, rh, + scratch_pool)); } } /* finalize */ - SVN_ERR(dgb__log_access(fs, &rs->rep_id, rh, SVN_FS_X__ITEM_TYPE_ANY_REP, + SVN_ERR(dbg__log_access(fs, &rs->rep_id, rh, SVN_FS_X__ITEM_TYPE_ANY_REP, scratch_pool)); rs->header_size = rh->header_size; @@ -820,8 +778,7 @@ svn_fs_x__check_rep(svn_fs_x__representation_t *rep, svn_revnum_t revision = svn_fs_x__get_revnum(rep->id.change_set); svn_fs_x__revision_file_t *rev_file; - SVN_ERR(svn_fs_x__open_pack_or_rev_file(&rev_file, fs, revision, - scratch_pool, scratch_pool)); + SVN_ERR(svn_fs_x__rev_file_init(&rev_file, fs, revision, scratch_pool)); /* Does REP->ID refer to an actual item? Which one is it? */ SVN_ERR(svn_fs_x__item_offset(&offset, &sub_item, fs, rev_file, &rep->id, @@ -1075,22 +1032,14 @@ get_cached_window_sizes(window_sizes_t **sizes, svn_boolean_t *is_cached, apr_pool_t *pool) { - if (! rs->window_cache) - { - /* txdelta window has not been enabled */ - *is_cached = FALSE; - } - else - { - svn_fs_x__window_cache_key_t key = { 0 }; - SVN_ERR(svn_cache__get_partial((void **)sizes, - is_cached, - rs->window_cache, - get_window_key(&key, rs), - get_cached_window_sizes_func, - NULL, - pool)); - } + svn_fs_x__window_cache_key_t key = { 0 }; + SVN_ERR(svn_cache__get_partial((void **)sizes, + is_cached, + rs->window_cache, + get_window_key(&key, rs), + get_cached_window_sizes_func, + NULL, + pool)); return SVN_NO_ERROR; } @@ -1103,33 +1052,25 @@ get_cached_window(svn_txdelta_window_t **window_p, apr_pool_t *result_pool, apr_pool_t *scratch_pool) { - if (! rs->window_cache) - { - /* txdelta window has not been enabled */ - *is_cached = FALSE; - } - else - { - /* ask the cache for the desired txdelta window */ - svn_fs_x__txdelta_cached_window_t *cached_window; - svn_fs_x__window_cache_key_t key = { 0 }; - get_window_key(&key, rs); - key.chunk_index = chunk_index; - SVN_ERR(svn_cache__get((void **) &cached_window, - is_cached, - rs->window_cache, - &key, - result_pool)); + /* ask the cache for the desired txdelta window */ + svn_fs_x__txdelta_cached_window_t *cached_window; + svn_fs_x__window_cache_key_t key = { 0 }; + get_window_key(&key, rs); + key.chunk_index = chunk_index; + SVN_ERR(svn_cache__get((void **) &cached_window, + is_cached, + rs->window_cache, + &key, + result_pool)); - if (*is_cached) - { - /* found it. Pass it back to the caller. */ - *window_p = cached_window->window; + if (*is_cached) + { + /* found it. Pass it back to the caller. */ + *window_p = cached_window->window; - /* manipulate the RS as if we just read the data */ - rs->current = cached_window->end_offset; - rs->chunk_index = chunk_index; - } + /* manipulate the RS as if we just read the data */ + rs->current = cached_window->end_offset; + rs->chunk_index = chunk_index; } return SVN_NO_ERROR; @@ -1145,23 +1086,20 @@ set_cached_window(svn_txdelta_window_t *window, apr_off_t start_offset, apr_pool_t *scratch_pool) { - if (rs->window_cache) - { - /* store the window and the first offset _past_ it */ - svn_fs_x__txdelta_cached_window_t cached_window; - svn_fs_x__window_cache_key_t key = {0}; + /* store the window and the first offset _past_ it */ + svn_fs_x__txdelta_cached_window_t cached_window; + svn_fs_x__window_cache_key_t key = {0}; - cached_window.window = window; - cached_window.start_offset = start_offset - rs->start; - cached_window.end_offset = rs->current; + cached_window.window = window; + cached_window.start_offset = start_offset - rs->start; + cached_window.end_offset = rs->current; - /* but key it with the start offset because that is the known state - * when we will look it up */ - SVN_ERR(svn_cache__set(rs->window_cache, - get_window_key(&key, rs), - &cached_window, - scratch_pool)); - } + /* but key it with the start offset because that is the known state + * when we will look it up */ + SVN_ERR(svn_cache__set(rs->window_cache, + get_window_key(&key, rs), + &cached_window, + scratch_pool)); return SVN_NO_ERROR; } @@ -1178,23 +1116,13 @@ get_cached_combined_window(svn_stringbuf_t **window_p, svn_boolean_t *is_cached, apr_pool_t *pool) { - if (! rs->combined_cache) - { - /* txdelta window has not been enabled */ - *is_cached = FALSE; - } - else - { - /* ask the cache for the desired txdelta window */ - svn_fs_x__window_cache_key_t key = { 0 }; - return svn_cache__get((void **)window_p, - is_cached, - rs->combined_cache, - get_window_key(&key, rs), - pool); - } - - return SVN_NO_ERROR; + /* ask the cache for the desired txdelta window */ + svn_fs_x__window_cache_key_t key = { 0 }; + return svn_cache__get((void **)window_p, + is_cached, + rs->combined_cache, + get_window_key(&key, rs), + pool); } /* Store the WINDOW read for the rep state RS in the current FSX session's @@ -1205,18 +1133,13 @@ set_cached_combined_window(svn_stringbuf_t *window, rep_state_t *rs, apr_pool_t *scratch_pool) { - if (rs->combined_cache) - { - /* but key it with the start offset because that is the known state - * when we will look it up */ - svn_fs_x__window_cache_key_t key = { 0 }; - return svn_cache__set(rs->combined_cache, - get_window_key(&key, rs), - window, - scratch_pool); - } - - return SVN_NO_ERROR; + /* but key it with the start offset because that is the known state + * when we will look it up */ + svn_fs_x__window_cache_key_t key = { 0 }; + return svn_cache__set(rs->combined_cache, + get_window_key(&key, rs), + window, + scratch_pool); } /* Build an array of rep_state structures in *LIST giving the delta @@ -1264,7 +1187,8 @@ build_rep_list(apr_array_header_t **list, /* for txn reps and containered reps, there won't be a cached * combined window */ if (svn_fs_x__is_revision(rep.id.change_set) - && rep_header->type != svn_fs_x__rep_container) + && rep_header->type != svn_fs_x__rep_container + && rs->combined_cache) SVN_ERR(get_cached_combined_window(window_p, rs, &is_cached, result_pool)); @@ -1364,30 +1288,37 @@ read_delta_window(svn_txdelta_window_t **nwin, int this_chunk, apr_off_t start_offset; apr_off_t end_offset; apr_pool_t *iterpool; + svn_stream_t *stream; + svn_fs_x__revision_file_t *file; + svn_boolean_t cacheable = rs->chunk_index == 0 + && svn_fs_x__is_revision(rs->rep_id.change_set) + && rs->window_cache; SVN_ERR_ASSERT(rs->chunk_index <= this_chunk); - SVN_ERR(dgb__log_access(rs->sfile->fs, &rs->rep_id, NULL, + SVN_ERR(dbg__log_access(rs->sfile->fs, &rs->rep_id, NULL, SVN_FS_X__ITEM_TYPE_ANY_REP, scratch_pool)); /* Read the next window. But first, try to find it in the cache. */ - SVN_ERR(get_cached_window(nwin, rs, this_chunk, &is_cached, - result_pool, scratch_pool)); - if (is_cached) - return SVN_NO_ERROR; + if (cacheable) + { + SVN_ERR(get_cached_window(nwin, rs, this_chunk, &is_cached, + result_pool, scratch_pool)); + if (is_cached) + return SVN_NO_ERROR; + } /* someone has to actually read the data from file. Open it */ SVN_ERR(auto_open_shared_file(rs->sfile)); + file = rs->sfile->rfile; /* invoke the 'block-read' feature for non-txn data. However, don't do that if we are in the middle of some representation, because the block is unlikely to contain other data. */ - if ( rs->chunk_index == 0 - && svn_fs_x__is_revision(rs->rep_id.change_set) - && rs->window_cache) + if (cacheable) { - SVN_ERR(block_read(NULL, rs->sfile->fs, &rs->rep_id, - rs->sfile->rfile, result_pool, scratch_pool)); + SVN_ERR(block_read(NULL, rs->sfile->fs, &rs->rep_id, file, NULL, + result_pool, scratch_pool)); /* reading the whole block probably also provided us with the desired txdelta window */ @@ -1405,18 +1336,19 @@ read_delta_window(svn_txdelta_window_t **nwin, int this_chunk, /* RS->FILE may be shared between RS instances -> make sure we point * to the right data. */ start_offset = rs->start + rs->current; - SVN_ERR(rs_aligned_seek(rs, NULL, start_offset, scratch_pool)); + SVN_ERR(svn_fs_x__rev_file_seek(file, NULL, start_offset)); /* Skip windows to reach the current chunk if we aren't there yet. */ iterpool = svn_pool_create(scratch_pool); while (rs->chunk_index < this_chunk) { - apr_file_t *file = rs->sfile->rfile->file; + apr_file_t *apr_file; svn_pool_clear(iterpool); - SVN_ERR(svn_txdelta_skip_svndiff_window(file, rs->ver, iterpool)); + SVN_ERR(svn_fs_x__rev_file_get(&apr_file, file)); + SVN_ERR(svn_txdelta_skip_svndiff_window(apr_file, rs->ver, iterpool)); rs->chunk_index++; - SVN_ERR(svn_fs_x__get_file_offset(&start_offset, file, iterpool)); + SVN_ERR(svn_io_file_get_offset(&start_offset, apr_file, iterpool)); rs->current = start_offset - rs->start; if (rs->current >= rs->size) @@ -1428,9 +1360,10 @@ read_delta_window(svn_txdelta_window_t **nwin, int this_chunk, svn_pool_destroy(iterpool); /* Actually read the next window. */ - SVN_ERR(svn_txdelta_read_svndiff_window(nwin, rs->sfile->rfile->stream, - rs->ver, result_pool)); - SVN_ERR(get_file_offset(&end_offset, rs, scratch_pool)); + SVN_ERR(svn_fs_x__rev_file_stream(&stream, file)); + SVN_ERR(svn_txdelta_read_svndiff_window(nwin, stream, rs->ver, + result_pool)); + SVN_ERR(svn_fs_x__rev_file_offset(&end_offset, file)); rs->current = end_offset - rs->start; if (rs->current > rs->size) return svn_error_create(SVN_ERR_FS_CORRUPT, NULL, @@ -1439,7 +1372,7 @@ read_delta_window(svn_txdelta_window_t **nwin, int this_chunk, /* the window has not been cached before, thus cache it now * (if caching is used for them at all) */ - if (svn_fs_x__is_revision(rs->rep_id.change_set)) + if (cacheable) SVN_ERR(set_cached_window(*nwin, rs, start_offset, scratch_pool)); return SVN_NO_ERROR; @@ -1458,31 +1391,29 @@ read_container_window(svn_stringbuf_t **nwin, svn_fs_x__data_t *ffd = fs->fsap_data; svn_fs_x__pair_cache_key_t key; svn_revnum_t revision = svn_fs_x__get_revnum(rs->rep_id.change_set); + svn_boolean_t is_cached = FALSE; + svn_fs_x__reps_baton_t baton; SVN_ERR(auto_set_start_offset(rs, scratch_pool)); key.revision = svn_fs_x__packed_base_rev(fs, revision); key.second = rs->start; /* already in cache? */ - if (ffd->reps_container_cache) - { - svn_boolean_t is_cached = FALSE; - svn_fs_x__reps_baton_t baton; - baton.fs = fs; - baton.idx = rs->sub_item; + baton.fs = fs; + baton.idx = rs->sub_item; - SVN_ERR(svn_cache__get_partial((void**)&extractor, &is_cached, - ffd->reps_container_cache, &key, - svn_fs_x__reps_get_func, &baton, - result_pool)); - } + SVN_ERR(svn_cache__get_partial((void**)&extractor, &is_cached, + ffd->reps_container_cache, &key, + svn_fs_x__reps_get_func, &baton, + result_pool)); /* read from disk, if necessary */ if (extractor == NULL) { SVN_ERR(auto_open_shared_file(rs->sfile)); SVN_ERR(block_read((void **)&extractor, fs, &rs->rep_id, - rs->sfile->rfile, result_pool, scratch_pool)); + rs->sfile->rfile, NULL, + result_pool, scratch_pool)); } SVN_ERR(svn_fs_x__extractor_drive(nwin, extractor, rs->current, size, @@ -1569,7 +1500,8 @@ get_combined_window(svn_stringbuf_t **result, single chunk. Only then will no other chunk need a deeper RS list than the cached chunk. */ if ( (rb->chunk_index == 0) && (rs->current == rs->size) - && svn_fs_x__is_revision(rs->rep_id.change_set)) + && svn_fs_x__is_revision(rs->rep_id.change_set) + && rs->combined_cache) SVN_ERR(set_cached_combined_window(buf, rs, new_pool)); rs->chunk_index++; @@ -1587,7 +1519,7 @@ get_combined_window(svn_stringbuf_t **result, } /* Returns whether or not the expanded fulltext of the file is cachable - * based on its size SIZE. The decision depends on the cache used by RB. + * based on its size SIZE. The decision depends on the cache used by FFD. */ static svn_boolean_t fulltext_size_is_cachable(svn_fs_x__data_t *ffd, @@ -1690,23 +1622,23 @@ cache_windows(svn_filesize_t *fulltext_len, else { svn_txdelta_window_t *window; + svn_fs_x__revision_file_t *file = rs->sfile->rfile; + svn_stream_t *stream; apr_off_t start_offset = rs->start + rs->current; apr_off_t end_offset; apr_off_t block_start; /* navigate to & read the current window */ - SVN_ERR(rs_aligned_seek(rs, &block_start, start_offset, iterpool)); - SVN_ERR(svn_txdelta_read_svndiff_window(&window, - rs->sfile->rfile->stream, - rs->ver, iterpool)); + SVN_ERR(svn_fs_x__rev_file_stream(&stream, file)); + SVN_ERR(svn_fs_x__rev_file_seek(file, &block_start, start_offset)); + SVN_ERR(svn_txdelta_read_svndiff_window(&window, stream, rs->ver, + iterpool)); /* aggregate expanded window size */ *fulltext_len += window->tview_len; /* determine on-disk window size */ - SVN_ERR(svn_fs_x__get_file_offset(&end_offset, - rs->sfile->rfile->file, - iterpool)); + SVN_ERR(svn_fs_x__rev_file_offset(&end_offset, rs->sfile->rfile)); rs->current = end_offset - rs->start; if (rs->current > rs->size) return svn_error_create(SVN_ERR_FS_CORRUPT, NULL, @@ -1735,25 +1667,22 @@ cache_windows(svn_filesize_t *fulltext_len, static svn_error_t * read_rep_header(svn_fs_x__rep_header_t **rep_header, svn_fs_t *fs, - svn_stream_t *stream, + svn_fs_x__revision_file_t *file, svn_fs_x__representation_cache_key_t *key, apr_pool_t *pool) { svn_fs_x__data_t *ffd = fs->fsap_data; + svn_stream_t *stream; svn_boolean_t is_cached = FALSE; - if (ffd->rep_header_cache) - { - SVN_ERR(svn_cache__get((void**)rep_header, &is_cached, - ffd->rep_header_cache, key, pool)); - if (is_cached) - return SVN_NO_ERROR; - } + SVN_ERR(svn_cache__get((void**)rep_header, &is_cached, + ffd->rep_header_cache, key, pool)); + if (is_cached) + return SVN_NO_ERROR; + SVN_ERR(svn_fs_x__rev_file_stream(&stream, file)); SVN_ERR(svn_fs_x__read_rep_header(rep_header, stream, pool, pool)); - - if (ffd->rep_header_cache) - SVN_ERR(svn_cache__set(ffd->rep_header_cache, key, *rep_header, pool)); + SVN_ERR(svn_cache__set(ffd->rep_header_cache, key, *rep_header, pool)); return SVN_NO_ERROR; } @@ -1779,8 +1708,7 @@ svn_fs_x__get_representation_length(svn_filesize_t *packed_len, key.revision = svn_fs_x__get_revnum(entry->items[0].change_set); key.is_packed = svn_fs_x__is_packed_rev(fs, key.revision); key.item_index = entry->items[0].number; - SVN_ERR(read_rep_header(&rep_header, fs, rev_file->stream, &key, - scratch_pool)); + SVN_ERR(read_rep_header(&rep_header, fs, rev_file, &key, scratch_pool)); /* prepare representation reader state (rs) structure */ SVN_ERR(init_rep_state(&rs, rep_header, fs, rev_file, entry, @@ -1829,10 +1757,10 @@ get_contents_from_windows(rep_read_baton_t *rb, This is where we need the pseudo rep_state created by build_rep_list(). */ apr_size_t offset = (apr_size_t)rs->current; - if (copy_len + offset > rb->base_window->len) - copy_len = offset < rb->base_window->len - ? rb->base_window->len - offset - : 0ul; + if (offset >= rb->base_window->len) + copy_len = 0ul; + else if (copy_len > rb->base_window->len - offset) + copy_len = rb->base_window->len - offset; memcpy (cur, rb->base_window->data + offset, copy_len); } @@ -2052,6 +1980,16 @@ skip_contents(rep_read_baton_t *baton, len -= to_read; buffer += to_read; } + + /* Make the MD5 calculation catch up with the data delivered + * (we did not run MD5 on the data that we took from the cache). */ + if (!err) + { + SVN_ERR(svn_checksum_update(baton->md5_checksum_ctx, + baton->current_fulltext->data, + baton->current_fulltext->len)); + baton->off += baton->current_fulltext->len; + } } else if (len > 0) { @@ -2067,6 +2005,15 @@ skip_contents(rep_read_baton_t *baton, err = get_contents_from_windows(baton, buffer, &to_read); len -= to_read; + + /* Make the MD5 calculation catch up with the data delivered + * (we did not run MD5 on the data that we took from the cache). */ + if (!err) + { + SVN_ERR(svn_checksum_update(baton->md5_checksum_ctx, + buffer, to_read)); + baton->off += to_read; + } } svn_pool_destroy(subpool); @@ -2112,8 +2059,13 @@ rep_read_contents(void *baton, SVN_ERR(skip_contents(rb, rb->fulltext_delivered)); } - /* Get the next block of data. */ - SVN_ERR(get_contents_from_windows(rb, buf, len)); + /* Get the next block of data. + * Keep in mind that the representation might be empty and leave us + * already positioned at the end of the rep. */ + if (rb->off == rb->len) + *len = 0; + else + SVN_ERR(get_contents_from_windows(rb, buf, len)); if (rb->current_fulltext) svn_stringbuf_appendbytes(rb->current_fulltext, buf, *len); @@ -2186,7 +2138,7 @@ svn_fs_x__get_contents(svn_stream_t **contents_p, /* Make the stream attempt fulltext cache lookups if the fulltext * is cacheable. If it is not, then also don't try to buffer and * cache it. */ - if (ffd->fulltext_cache && cache_fulltext + if ( cache_fulltext && SVN_IS_VALID_REVNUM(revision) && fulltext_size_is_cachable(ffd, len)) { @@ -2208,6 +2160,86 @@ svn_fs_x__get_contents(svn_stream_t **contents_p, return SVN_NO_ERROR; } +svn_error_t * +svn_fs_x__get_contents_from_file(svn_stream_t **contents_p, + svn_fs_t *fs, + svn_fs_x__representation_t *rep, + apr_file_t *file, + apr_off_t offset, + apr_pool_t *pool) +{ + rep_read_baton_t *rb; + svn_fs_x__pair_cache_key_t fulltext_cache_key = { SVN_INVALID_REVNUM, 0 }; + rep_state_t *rs = apr_pcalloc(pool, sizeof(*rs)); + svn_fs_x__rep_header_t *rh; + svn_stream_t *stream; + + /* Initialize the reader baton. Some members may added lazily + * while reading from the stream. */ + SVN_ERR(rep_read_get_baton(&rb, fs, rep, fulltext_cache_key, pool)); + + /* Continue constructing RS. Leave caches as NULL. */ + rs->size = rep->size; + rs->rep_id = rep->id; + rs->ver = -1; + rs->start = -1; + + /* Provide just enough file access info to allow for a basic read from + * FILE but leave all index / footer info with empty values b/c FILE + * probably is not a complete revision file. */ + rs->sfile = apr_pcalloc(pool, sizeof(*rs->sfile)); + rs->sfile->revision = SVN_INVALID_REVNUM; + rs->sfile->pool = pool; + rs->sfile->fs = fs; + SVN_ERR(svn_fs_x__rev_file_wrap_temp(&rs->sfile->rfile, fs, file, pool)); + + /* Read the rep header. */ + SVN_ERR(svn_fs_x__rev_file_seek(rs->sfile->rfile, NULL, offset)); + SVN_ERR(svn_fs_x__rev_file_stream(&stream, rs->sfile->rfile)); + SVN_ERR(svn_fs_x__read_rep_header(&rh, stream, pool, pool)); + SVN_ERR(svn_fs_x__rev_file_offset(&rs->start, rs->sfile->rfile)); + rs->header_size = rh->header_size; + + /* Log the access. */ + SVN_ERR(dbg__log_access(fs, &rep->id, rh, + SVN_FS_X__ITEM_TYPE_ANY_REP, pool)); + + /* Build the representation list (delta chain). */ + if (rh->type == svn_fs_x__rep_self_delta) + { + rb->rs_list = apr_array_make(pool, 1, sizeof(rep_state_t *)); + APR_ARRAY_PUSH(rb->rs_list, rep_state_t *) = rs; + rb->src_state = NULL; + } + else + { + svn_fs_x__representation_t next_rep = { 0 }; + + /* skip "SVNx" diff marker */ + rs->current = 4; + + /* REP's base rep is inside a proper revision. + * It can be reconstructed in the usual way. */ + next_rep.id.change_set = svn_fs_x__change_set_by_rev(rh->base_revision); + next_rep.id.number = rh->base_item_index; + next_rep.size = rh->base_length; + + SVN_ERR(build_rep_list(&rb->rs_list, &rb->base_window, + &rb->src_state, rb->fs, &next_rep, + rb->filehandle_pool, rb->scratch_pool)); + + /* Insert the access to REP as the first element of the delta chain. */ + svn_sort__array_insert(rb->rs_list, &rs, 0); + } + + /* Now, the baton is complete and we can assemble the stream around it. */ + *contents_p = svn_stream_create(rb, pool); + svn_stream_set_read2(*contents_p, NULL /* only full read support */, + rep_read_contents); + svn_stream_set_close(*contents_p, rep_read_contents_close); + + return SVN_NO_ERROR; +} /* Baton for cache_access_wrapper. Wraps the original parameters of * svn_fs_x__try_process_file_content(). @@ -2257,8 +2289,7 @@ svn_fs_x__try_process_file_contents(svn_boolean_t *success, fulltext_cache_key.revision = svn_fs_x__get_revnum(rep->id.change_set); fulltext_cache_key.second = rep->id.number; - if (ffd->fulltext_cache - && SVN_IS_VALID_REVNUM(fulltext_cache_key.revision) + if ( SVN_IS_VALID_REVNUM(fulltext_cache_key.revision) && fulltext_size_is_cachable(ffd, rep->expanded_size)) { cache_access_wrapper_baton_t wrapper_baton; @@ -2344,12 +2375,11 @@ svn_fs_x__get_file_delta_stream(svn_txdelta_stream_t **stream_p, svn_stream_t *source_stream, *target_stream; rep_state_t *rep_state; svn_fs_x__rep_header_t *rep_header; - svn_fs_x__data_t *ffd = fs->fsap_data; /* Try a shortcut: if the target is stored as a delta against the source, then just use that delta. However, prefer using the fulltext cache whenever that is available. */ - if (target->data_rep && (source || !ffd->fulltext_cache)) + if (target->data_rep && source) { /* Read target's base rep if any. */ SVN_ERR(create_rep_state(&rep_state, &rep_header, NULL, @@ -2449,117 +2479,147 @@ compare_dirent_name(const void *a, return strcmp(lhs->name, rhs); } -/* Into ENTRIES, read all directories entries from the key-value text in - * STREAM. If INCREMENTAL is TRUE, read until the end of the STREAM and +/* Into ENTRIES, parse all directories entries from the serialized form in + * DATA. If INCREMENTAL is TRUE, read until the end of the STREAM and * update the data. ID is provided for nicer error messages. + * + * The contents of DATA will be shared with the items in ENTRIES, i.e. it + * must not be modified afterwards and must remain valid as long as ENTRIES + * is valid. Use SCRATCH_POOL for temporary allocations. */ static svn_error_t * -read_dir_entries(apr_array_header_t *entries, - svn_stream_t *stream, - svn_boolean_t incremental, - const svn_fs_x__id_t *id, - apr_pool_t *result_pool, - apr_pool_t *scratch_pool) +parse_dir_entries(apr_array_header_t **entries_p, + const svn_stringbuf_t *data, + svn_boolean_t incremental, + const svn_fs_x__id_t *id, + apr_pool_t *result_pool, + apr_pool_t *scratch_pool) { - apr_pool_t *iterpool = svn_pool_create(scratch_pool); + const apr_byte_t *p = (const apr_byte_t *)data->data; + const apr_byte_t *end = p + data->len; + apr_uint64_t count; apr_hash_t *hash = incremental ? svn_hash__make(scratch_pool) : NULL; - const char *terminator = SVN_HASH_TERMINATOR; - - /* Read until the terminator (non-incremental) or the end of STREAM - (incremental mode). In the latter mode, we use a temporary HASH - to make updating and removing entries cheaper. */ - while (1) - { - svn_hash__entry_t entry; - svn_fs_x__dirent_t *dirent; - char *str; - - svn_pool_clear(iterpool); - SVN_ERR(svn_hash__read_entry(&entry, stream, terminator, - incremental, iterpool)); + apr_array_header_t *entries; - /* End of directory? */ - if (entry.key == NULL) - { - /* In incremental mode, we skip the terminator and read the - increments following it until the end of the stream. */ - if (incremental && terminator) - terminator = NULL; - else - break; - } + /* Construct the resulting container. */ + p = svn__decode_uint(&count, p, end); + if (count > INT_MAX) + return svn_error_createf(SVN_ERR_FS_CORRUPT, NULL, + _("Directory for '%s' is too large"), + svn_fs_x__id_unparse(id, scratch_pool)->data); - /* Deleted entry? */ - if (entry.val == NULL) - { - /* We must be in incremental mode */ - assert(hash); - apr_hash_set(hash, entry.key, entry.keylen, NULL); - continue; - } + entries = apr_array_make(result_pool, (int)count, + sizeof(svn_fs_x__dirent_t *)); - /* Add a new directory entry. */ + while (p != end) + { + apr_size_t len; + svn_fs_x__dirent_t *dirent; dirent = apr_pcalloc(result_pool, sizeof(*dirent)); - dirent->name = apr_pstrmemdup(result_pool, entry.key, entry.keylen); - str = svn_cstring_tokenize(" ", &entry.val); - if (str == NULL) + /* The part of the serialized entry that is not the name will be + * about 6 bytes or less. Since APR allocates with an 8 byte + * alignment (4 bytes loss on average per string), simply using + * the name string in DATA already gives us near-optimal memory + * usage. */ + dirent->name = (const char *)p; + len = strlen(dirent->name); + p += len + 1; + if (p == end) return svn_error_createf(SVN_ERR_FS_CORRUPT, NULL, - _("Directory entry corrupt in '%s'"), - svn_fs_x__id_unparse(id, scratch_pool)->data); + _("Directory entry missing kind in '%s'"), + svn_fs_x__id_unparse(id, scratch_pool)->data); - if (strcmp(str, SVN_FS_X__KIND_FILE) == 0) - { - dirent->kind = svn_node_file; - } - else if (strcmp(str, SVN_FS_X__KIND_DIR) == 0) - { - dirent->kind = svn_node_dir; - } - else - { - return svn_error_createf(SVN_ERR_FS_CORRUPT, NULL, - _("Directory entry corrupt in '%s'"), - svn_fs_x__id_unparse(id, scratch_pool)->data); - } + dirent->kind = (svn_node_kind_t)*(p++); + if (p == end) + return svn_error_createf(SVN_ERR_FS_CORRUPT, NULL, + _("Directory entry missing change set in '%s'"), + svn_fs_x__id_unparse(id, scratch_pool)->data); - str = svn_cstring_tokenize(" ", &entry.val); - if (str == NULL) + p = svn__decode_int(&dirent->id.change_set, p, end); + if (p == end) return svn_error_createf(SVN_ERR_FS_CORRUPT, NULL, - _("Directory entry corrupt in '%s'"), - svn_fs_x__id_unparse(id, scratch_pool)->data); + _("Directory entry missing item number in '%s'"), + svn_fs_x__id_unparse(id, scratch_pool)->data); - SVN_ERR(svn_fs_x__id_parse(&dirent->id, str)); + p = svn__decode_uint(&dirent->id.number, p, end); /* In incremental mode, update the hash; otherwise, write to the * final array. */ if (incremental) - apr_hash_set(hash, dirent->name, entry.keylen, dirent); + { + /* Insertion / update or a deletion? */ + if (svn_fs_x__id_used(&dirent->id)) + apr_hash_set(hash, dirent->name, len, dirent); + else + apr_hash_set(hash, dirent->name, len, NULL); + } else - APR_ARRAY_PUSH(entries, svn_fs_x__dirent_t *) = dirent; + { + APR_ARRAY_PUSH(entries, svn_fs_x__dirent_t *) = dirent; + } } - /* Convert container to a sorted array. */ if (incremental) { + /* Convert container into a sorted array. */ apr_hash_index_t *hi; - for (hi = apr_hash_first(iterpool, hash); hi; hi = apr_hash_next(hi)) + for (hi = apr_hash_first(scratch_pool, hash); hi; hi = apr_hash_next(hi)) APR_ARRAY_PUSH(entries, svn_fs_x__dirent_t *) = apr_hash_this_val(hi); + + if (!sorted(entries)) + svn_sort__array(entries, compare_dirents); + } + else + { + /* Check that we read the expected amount of entries. */ + if ((apr_uint64_t)entries->nelts != count) + return svn_error_createf(SVN_ERR_FS_CORRUPT, NULL, + _("Directory length mismatch in '%s'"), + svn_fs_x__id_unparse(id, scratch_pool)->data); } - if (!sorted(entries)) - svn_sort__array(entries, compare_dirents); + *entries_p = entries; - svn_pool_destroy(iterpool); + return SVN_NO_ERROR; +} + +/* For directory NODEREV in FS, return the *FILESIZE of its in-txn + * representation. If the directory representation is comitted data, + * set *FILESIZE to SVN_INVALID_FILESIZE. Use SCRATCH_POOL for temporaries. + */ +static svn_error_t * +get_txn_dir_info(svn_filesize_t *filesize, + svn_fs_t *fs, + svn_fs_x__noderev_t *noderev, + apr_pool_t *scratch_pool) +{ + if (noderev->data_rep + && ! svn_fs_x__is_revision(noderev->data_rep->id.change_set)) + { + const svn_io_dirent2_t *dirent; + const char *filename; + + filename = svn_fs_x__path_txn_node_children(fs, &noderev->noderev_id, + scratch_pool, scratch_pool); + + SVN_ERR(svn_io_stat_dirent2(&dirent, filename, FALSE, FALSE, + scratch_pool, scratch_pool)); + *filesize = dirent->filesize; + } + else + { + *filesize = SVN_INVALID_FILESIZE; + } return SVN_NO_ERROR; } -/* Fetch the contents of a directory into ENTRIES. Values are stored +/* Fetch the contents of a directory into DIR. Values are stored as filename to string mappings; further conversion is necessary to convert them into svn_fs_x__dirent_t values. */ static svn_error_t * -get_dir_contents(apr_array_header_t **entries, +get_dir_contents(svn_fs_x__dir_data_t *dir, svn_fs_t *fs, svn_fs_x__noderev_t *noderev, apr_pool_t *result_pool, @@ -2567,44 +2627,67 @@ get_dir_contents(apr_array_header_t **entries, { svn_stream_t *contents; const svn_fs_x__id_t *id = &noderev->noderev_id; + apr_size_t len; + svn_stringbuf_t *text; + svn_boolean_t incremental; - *entries = apr_array_make(result_pool, 16, sizeof(svn_fs_x__dirent_t *)); + /* Initialize the result. */ + dir->txn_filesize = SVN_INVALID_FILESIZE; + + /* Read dir contents - unless there is none in which case we are done. */ if (noderev->data_rep && ! svn_fs_x__is_revision(noderev->data_rep->id.change_set)) { - const char *filename - = svn_fs_x__path_txn_node_children(fs, id, scratch_pool, - scratch_pool); + /* Get location & current size of the directory representation. */ + const char *filename; + apr_file_t *file; + + filename = svn_fs_x__path_txn_node_children(fs, id, scratch_pool, + scratch_pool); /* The representation is mutable. Read the old directory contents from the mutable children file, followed by the changes we've made in this transaction. */ - SVN_ERR(svn_stream_open_readonly(&contents, filename, scratch_pool, - scratch_pool)); - SVN_ERR(read_dir_entries(*entries, contents, TRUE, id, - result_pool, scratch_pool)); - SVN_ERR(svn_stream_close(contents)); + SVN_ERR(svn_io_file_open(&file, filename, APR_READ | APR_BUFFERED, + APR_OS_DEFAULT, scratch_pool)); + + /* Obtain txn children file size. */ + SVN_ERR(svn_io_file_size_get(&dir->txn_filesize, file, scratch_pool)); + len = (apr_size_t)dir->txn_filesize; + + /* Finally, provide stream access to FILE. */ + contents = svn_stream_from_aprfile2(file, FALSE, scratch_pool); + incremental = TRUE; } else if (noderev->data_rep) { - /* Undeltify content before parsing it. Otherwise, we could only - * parse it byte-by-byte. - */ - apr_size_t len = noderev->data_rep->expanded_size; - svn_stringbuf_t *text; - /* The representation is immutable. Read it normally. */ + len = noderev->data_rep->expanded_size; SVN_ERR(svn_fs_x__get_contents(&contents, fs, noderev->data_rep, FALSE, scratch_pool)); - SVN_ERR(svn_stringbuf_from_stream(&text, contents, len, scratch_pool)); - SVN_ERR(svn_stream_close(contents)); - - /* de-serialize hash */ - contents = svn_stream_from_stringbuf(text, scratch_pool); - SVN_ERR(read_dir_entries(*entries, contents, FALSE, id, - result_pool, scratch_pool)); + incremental = FALSE; + } + else + { + /* Empty representation == empty directory. */ + dir->entries = apr_array_make(result_pool, 0, + sizeof(svn_fs_x__dirent_t *)); + return SVN_NO_ERROR; } + /* Read the whole stream contents into a single buffer. + * Due to our LEN hint, no allocation overhead occurs. + * + * Also, a large portion of TEXT will be file / dir names which we + * directly reference from DIR->ENTRIES instead of copying them. + * Hence, we need to use the RESULT_POOL here. */ + SVN_ERR(svn_stringbuf_from_stream(&text, contents, len, result_pool)); + SVN_ERR(svn_stream_close(contents)); + + /* de-serialize hash */ + SVN_ERR(parse_dir_entries(&dir->entries, text, incremental, id, + result_pool, scratch_pool)); + return SVN_NO_ERROR; } @@ -2618,26 +2701,24 @@ locate_dir_cache(svn_fs_t *fs, svn_fs_x__noderev_t *noderev) { svn_fs_x__data_t *ffd = fs->fsap_data; - if (svn_fs_x__is_txn(noderev->noderev_id.change_set)) + + if (!noderev->data_rep) { - /* data in txns must be addressed by ID since the representation has - not been created, yet. */ + /* no data rep -> empty directory. + Use a key that does definitely not clash with non-NULL reps. */ + key->change_set = SVN_FS_X__INVALID_CHANGE_SET; + key->number = SVN_FS_X__ITEM_INDEX_UNUSED; + } + else if (svn_fs_x__is_txn(noderev->noderev_id.change_set)) + { + /* data in txns must be addressed by noderev ID since the + representation has not been created, yet. */ *key = noderev->noderev_id; } else { /* committed data can use simple rev,item pairs */ - if (noderev->data_rep) - { - *key = noderev->data_rep->id; - } - else - { - /* no data rep -> empty directory. - Use a key that does definitely not clash with non-NULL reps. */ - key->change_set = SVN_FS_X__INVALID_CHANGE_SET; - key->number = SVN_FS_X__ITEM_INDEX_UNUSED; - } + *key = noderev->data_rep->id; } return ffd->dir_cache; @@ -2651,26 +2732,40 @@ svn_fs_x__rep_contents_dir(apr_array_header_t **entries_p, apr_pool_t *scratch_pool) { svn_fs_x__id_t key; + svn_fs_x__dir_data_t *dir; /* find the cache we may use */ svn_cache__t *cache = locate_dir_cache(fs, &key, noderev); - if (cache) + svn_boolean_t found; + + SVN_ERR(svn_cache__get((void **)&dir, &found, cache, &key, result_pool)); + if (found) { - svn_boolean_t found; + /* Verify that the cached dir info is not stale + * (no-op for committed data). */ + svn_filesize_t filesize; + SVN_ERR(get_txn_dir_info(&filesize, fs, noderev, scratch_pool)); - SVN_ERR(svn_cache__get((void **)entries_p, &found, cache, &key, - result_pool)); - if (found) - return SVN_NO_ERROR; + if (filesize == dir->txn_filesize) + { + /* Still valid. Done. */ + *entries_p = dir->entries; + return SVN_NO_ERROR; + } } /* Read in the directory contents. */ - SVN_ERR(get_dir_contents(entries_p, fs, noderev, result_pool, - scratch_pool)); + dir = apr_pcalloc(scratch_pool, sizeof(*dir)); + SVN_ERR(get_dir_contents(dir, fs, noderev, result_pool, scratch_pool)); + *entries_p = dir->entries; - /* Update the cache, if we are to use one. */ - if (cache) - SVN_ERR(svn_cache__set(cache, &key, *entries_p, scratch_pool)); + /* Update the cache, if we are to use one. + * + * Don't even attempt to serialize very large directories; it would cause + * an unnecessary memory allocation peak. 100 bytes/entry is about right. + */ + if (svn_cache__is_cachable(cache, 100 * dir->entries->nelts)) + SVN_ERR(svn_cache__set(cache, &key, dir, scratch_pool)); return SVN_NO_ERROR; } @@ -2699,40 +2794,49 @@ svn_fs_x__rep_contents_dir_entry(svn_fs_x__dirent_t **dirent, /* find the cache we may use */ svn_fs_x__id_t key; svn_cache__t *cache = locate_dir_cache(fs, &key, noderev); - if (cache) - { - svn_fs_x__ede_baton_t baton; - baton.hint = *hint; - baton.name = name; - - /* Cache lookup. */ - SVN_ERR(svn_cache__get_partial((void **)dirent, - &found, - cache, - &key, - svn_fs_x__extract_dir_entry, - &baton, - result_pool)); - - /* Remember the new clue only if we found something at that spot. */ - if (found) - *hint = baton.hint; - } + svn_fs_x__ede_baton_t baton; + + svn_filesize_t filesize; + SVN_ERR(get_txn_dir_info(&filesize, fs, noderev, scratch_pool)); + + /* Cache lookup. */ + baton.hint = *hint; + baton.name = name; + baton.txn_filesize = filesize; + + SVN_ERR(svn_cache__get_partial((void **)dirent, + &found, + cache, + &key, + svn_fs_x__extract_dir_entry, + &baton, + result_pool)); + + /* Remember the new clue only if we found something at that spot. */ + if (found) + *hint = baton.hint; /* fetch data from disk if we did not find it in the cache */ - if (! found) + if (! found || baton.out_of_date) { - apr_array_header_t *entries; svn_fs_x__dirent_t *entry; svn_fs_x__dirent_t *entry_copy = NULL; + svn_fs_x__dir_data_t dir; + + /* Read in the directory contents. */ + SVN_ERR(get_dir_contents(&dir, fs, noderev, scratch_pool, + scratch_pool)); - /* read the dir from the file system. It will probably be put it - into the cache for faster lookup in future calls. */ - SVN_ERR(svn_fs_x__rep_contents_dir(&entries, fs, noderev, - scratch_pool, scratch_pool)); + /* Update the cache, if we are to use one. + * + * Don't even attempt to serialize very large directories; it would + * cause an unnecessary memory allocation peak. 150 bytes / entry is + * about right. */ + if (cache && svn_cache__is_cachable(cache, 150 * dir.entries->nelts)) + SVN_ERR(svn_cache__set(cache, &key, &dir, scratch_pool)); /* find desired entry and return a copy in POOL, if found */ - entry = svn_fs_x__find_dir_entry(entries, name, NULL); + entry = svn_fs_x__find_dir_entry(dir.entries, name, NULL); if (entry) { entry_copy = apr_pmemdup(result_pool, entry, sizeof(*entry_copy)); @@ -2746,140 +2850,168 @@ svn_fs_x__rep_contents_dir_entry(svn_fs_x__dirent_t **dirent, } svn_error_t * -svn_fs_x__get_proplist(apr_hash_t **proplist_p, +svn_fs_x__get_proplist(apr_hash_t **proplist, svn_fs_t *fs, svn_fs_x__noderev_t *noderev, apr_pool_t *result_pool, apr_pool_t *scratch_pool) { - apr_hash_t *proplist; svn_stream_t *stream; const svn_fs_x__id_t *noderev_id = &noderev->noderev_id; if (noderev->prop_rep && !svn_fs_x__is_revision(noderev->prop_rep->id.change_set)) { + svn_stringbuf_t *content; + svn_string_t *as_string; const char *filename = svn_fs_x__path_txn_node_props(fs, noderev_id, scratch_pool, scratch_pool); - proplist = apr_hash_make(result_pool); + SVN_ERR(svn_stringbuf_from_file2(&content, filename, result_pool)); - SVN_ERR(svn_stream_open_readonly(&stream, filename, scratch_pool, - scratch_pool)); - SVN_ERR(svn_hash_read2(proplist, stream, SVN_HASH_TERMINATOR, - result_pool)); - SVN_ERR(svn_stream_close(stream)); + as_string = svn_stringbuf__morph_into_string(content); + SVN_ERR_W(svn_fs_x__parse_properties(proplist, as_string, result_pool), + apr_psprintf(scratch_pool, + "malformed property list for node-revision '%s' in '%s'", + svn_fs_x__id_unparse(&noderev->noderev_id, + scratch_pool)->data, + filename)); } else if (noderev->prop_rep) { svn_fs_x__data_t *ffd = fs->fsap_data; svn_fs_x__representation_t *rep = noderev->prop_rep; svn_fs_x__pair_cache_key_t key = { 0 }; + svn_string_t *content; + svn_boolean_t is_cached; key.revision = svn_fs_x__get_revnum(rep->id.change_set); key.second = rep->id.number; - if (ffd->properties_cache && SVN_IS_VALID_REVNUM(key.revision)) - { - svn_boolean_t is_cached; - SVN_ERR(svn_cache__get((void **) proplist_p, &is_cached, - ffd->properties_cache, &key, result_pool)); - if (is_cached) - return SVN_NO_ERROR; - } + SVN_ERR(svn_cache__get((void **) proplist, &is_cached, + ffd->properties_cache, &key, result_pool)); + if (is_cached) + return SVN_NO_ERROR; - proplist = apr_hash_make(result_pool); - SVN_ERR(svn_fs_x__get_contents(&stream, fs, noderev->prop_rep, FALSE, - scratch_pool)); - SVN_ERR(svn_hash_read2(proplist, stream, SVN_HASH_TERMINATOR, - result_pool)); - SVN_ERR(svn_stream_close(stream)); + SVN_ERR(svn_fs_x__get_contents(&stream, fs, rep, FALSE, scratch_pool)); + SVN_ERR(svn_string_from_stream2(&content, stream, rep->expanded_size, + result_pool)); - if (ffd->properties_cache && SVN_IS_VALID_REVNUM(rep->id.change_set)) - SVN_ERR(svn_cache__set(ffd->properties_cache, &key, proplist, - scratch_pool)); + SVN_ERR_W(svn_fs_x__parse_properties(proplist, content, result_pool), + apr_psprintf(scratch_pool, + "malformed property list for node-revision '%s'", + svn_fs_x__id_unparse(&noderev->noderev_id, + scratch_pool)->data)); + + SVN_ERR(svn_cache__set(ffd->properties_cache, &key, *proplist, + scratch_pool)); } else { /* return an empty prop list if the node doesn't have any props */ - proplist = apr_hash_make(result_pool); + *proplist = apr_hash_make(result_pool); } - *proplist_p = proplist; - return SVN_NO_ERROR; } +svn_error_t * +svn_fs_x__create_changes_context(svn_fs_x__changes_context_t **context, + svn_fs_t *fs, + svn_revnum_t rev, + apr_pool_t *result_pool, + apr_pool_t *scratch_pool) +{ + svn_fs_x__changes_context_t *result = apr_pcalloc(result_pool, + sizeof(*result)); + result->fs = fs; + result->revision = rev; + SVN_ERR(svn_fs_x__ensure_revision_exists(rev, fs, scratch_pool)); + SVN_ERR(svn_fs_x__rev_file_init(&result->revision_file, fs, rev, + result_pool)); + + *context = result; + return SVN_NO_ERROR; +} svn_error_t * svn_fs_x__get_changes(apr_array_header_t **changes, - svn_fs_t *fs, - svn_revnum_t rev, - apr_pool_t *result_pool) + svn_fs_x__changes_context_t *context, + apr_pool_t *result_pool, + apr_pool_t *scratch_pool) { - svn_fs_x__revision_file_t *revision_file; svn_boolean_t found; - svn_fs_x__data_t *ffd = fs->fsap_data; - apr_pool_t *scratch_pool = svn_pool_create(result_pool); + svn_fs_x__data_t *ffd = context->fs->fsap_data; svn_fs_x__id_t id; - id.change_set = svn_fs_x__change_set_by_rev(rev); + id.change_set = svn_fs_x__change_set_by_rev(context->revision); id.number = SVN_FS_X__ITEM_INDEX_CHANGES; - /* Provide revision file. */ - - SVN_ERR(svn_fs_x__ensure_revision_exists(rev, fs, scratch_pool)); - SVN_ERR(svn_fs_x__open_pack_or_rev_file(&revision_file, fs, rev, - scratch_pool, scratch_pool)); - /* try cache lookup first */ - if (ffd->changes_container_cache && svn_fs_x__is_packed_rev(fs, rev)) + if (svn_fs_x__is_packed_rev(context->fs, context->revision)) { apr_off_t offset; - apr_uint32_t sub_item; svn_fs_x__pair_cache_key_t key; + svn_fs_x__changes_get_list_baton_t baton; + baton.start = (int)context->next; + baton.eol = &context->eol; - SVN_ERR(svn_fs_x__item_offset(&offset, &sub_item, fs, revision_file, + SVN_ERR(svn_fs_x__item_offset(&offset, &baton.sub_item, context->fs, + context->revision_file, &id, scratch_pool)); - key.revision = svn_fs_x__packed_base_rev(fs, rev); + key.revision = svn_fs_x__packed_base_rev(context->fs, + context->revision); key.second = offset; SVN_ERR(svn_cache__get_partial((void **)changes, &found, ffd->changes_container_cache, &key, svn_fs_x__changes_get_list_func, - &sub_item, result_pool)); - } - else if (ffd->changes_cache) - { - SVN_ERR(svn_cache__get((void **) changes, &found, ffd->changes_cache, - &rev, result_pool)); + &baton, result_pool)); } else { - found = FALSE; + svn_fs_x__changes_list_t *changes_list; + svn_fs_x__pair_cache_key_t key; + key.revision = context->revision; + key.second = context->next; + + SVN_ERR(svn_cache__get((void **)&changes_list, &found, + ffd->changes_cache, &key, result_pool)); + + if (found) + { + /* Where to look next - if there is more data. */ + context->eol = changes_list->eol; + context->next_offset = changes_list->end_offset; + + /* Return the block as a "proper" APR array. */ + (*changes) = apr_array_make(result_pool, 0, sizeof(void *)); + (*changes)->elts = (char *)changes_list->changes; + (*changes)->nelts = changes_list->count; + (*changes)->nalloc = changes_list->count; + } } if (!found) { /* 'block-read' will also provide us with the desired data */ - SVN_ERR(block_read((void **)changes, fs, &id, revision_file, + SVN_ERR(block_read((void **)changes, context->fs, &id, + context->revision_file, context, result_pool, scratch_pool)); - - SVN_ERR(svn_fs_x__close_revision_file(revision_file)); } - SVN_ERR(dgb__log_access(fs, &id, *changes, SVN_FS_X__ITEM_TYPE_CHANGES, - scratch_pool)); + context->next += (*changes)->nelts; + + SVN_ERR(dbg__log_access(context->fs, &id, *changes, + SVN_FS_X__ITEM_TYPE_CHANGES, scratch_pool)); - svn_pool_destroy(scratch_pool); return SVN_NO_ERROR; } /* Fetch the representation data (header, txdelta / plain windows) - * addressed by ENTRY->ITEM in FS and cache it if caches are enabled. - * Read the data from the already open FILE and the wrapping - * STREAM object. If MAX_OFFSET is not -1, don't read windows that start + * addressed by ENTRY->ITEM in FS and cache it under KEY. Read the data + * from REV_FILE. If MAX_OFFSET is not -1, don't read windows that start * at or beyond that offset. Use SCRATCH_POOL for temporary allocations. */ static svn_error_t * @@ -2890,20 +3022,16 @@ block_read_contents(svn_fs_t *fs, apr_off_t max_offset, apr_pool_t *scratch_pool) { - svn_fs_x__data_t *ffd = fs->fsap_data; svn_fs_x__representation_cache_key_t header_key = { 0 }; rep_state_t rs = { 0 }; svn_filesize_t fulltext_len; svn_fs_x__rep_header_t *rep_header; - if (!ffd->txdelta_window_cache || !ffd->combined_window_cache) - return SVN_NO_ERROR; - header_key.revision = (apr_int32_t)key->revision; header_key.is_packed = svn_fs_x__is_packed_rev(fs, header_key.revision); header_key.item_index = key->second; - SVN_ERR(read_rep_header(&rep_header, fs, rev_file->stream, &header_key, + SVN_ERR(read_rep_header(&rep_header, fs, rev_file, &header_key, scratch_pool)); SVN_ERR(init_rep_state(&rs, rep_header, fs, rev_file, entry, scratch_pool)); SVN_ERR(cache_windows(&fulltext_len, fs, &rs, max_offset, scratch_pool)); @@ -2913,28 +3041,28 @@ block_read_contents(svn_fs_t *fs, /* For the given REV_FILE in FS, in *STREAM return a stream covering the * item specified by ENTRY. Also, verify the item's content by low-level - * checksum. Allocate the result in POOL. + * checksum. Allocate the result in RESULT_POOL. */ static svn_error_t * read_item(svn_stream_t **stream, svn_fs_t *fs, svn_fs_x__revision_file_t *rev_file, svn_fs_x__p2l_entry_t* entry, - apr_pool_t *pool) + apr_pool_t *result_pool) { apr_uint32_t digest; svn_checksum_t *expected, *actual; apr_uint32_t plain_digest; + svn_stringbuf_t *text; /* Read item into string buffer. */ - svn_stringbuf_t *text = svn_stringbuf_create_ensure(entry->size, pool); + text = svn_stringbuf_create_ensure(entry->size, result_pool); text->len = entry->size; text->data[text->len] = 0; - SVN_ERR(svn_io_file_read_full2(rev_file->file, text->data, text->len, - NULL, NULL, pool)); + SVN_ERR(svn_fs_x__rev_file_read(rev_file, text->data, text->len)); /* Return (construct, calculate) stream and checksum. */ - *stream = svn_stream_from_stringbuf(text, pool); + *stream = svn_stream_from_stringbuf(text, result_pool); digest = svn__fnv1a_32x4(text->data, text->len); /* Checksums will match most of the time. */ @@ -2945,84 +3073,114 @@ read_item(svn_stream_t **stream, * nice error messages. */ plain_digest = htonl(entry->fnv1_checksum); expected = svn_checksum__from_digest_fnv1a_32x4( - (const unsigned char *)&plain_digest, pool); + (const unsigned char *)&plain_digest, result_pool); plain_digest = htonl(digest); actual = svn_checksum__from_digest_fnv1a_32x4( - (const unsigned char *)&plain_digest, pool); + (const unsigned char *)&plain_digest, result_pool); /* Construct the full error message with all the info we have. */ - return svn_checksum_mismatch_err(expected, actual, pool, + return svn_checksum_mismatch_err(expected, actual, result_pool, _("Low-level checksum mismatch while reading\n" "%s bytes of meta data at offset %s "), - apr_psprintf(pool, "%" APR_OFF_T_FMT, entry->size), - apr_psprintf(pool, "%" APR_OFF_T_FMT, entry->offset)); + apr_off_t_toa(result_pool, entry->size), + apr_off_t_toa(result_pool, entry->offset)); } -/* Read all txdelta / plain windows following REP_HEADER in FS as described - * by ENTRY. Read the data from the already open FILE and the wrapping - * STREAM object. If MAX_OFFSET is not -1, don't read windows that start - * at or beyond that offset. Use SCRATCH_POOL for temporary allocations. - * If caching is not enabled, this is a no-op. +/* If not already cached or if MUST_READ is set, read the changed paths + * list addressed by ENTRY in FS and retúrn it in *CHANGES. Cache the + * result if caching is enabled. Read the data from REV_FILE. Trim the + * data in *CHANGES to the range given by CONTEXT. Allocate *CHANGES in + * RESUSLT_POOL and allocate temporaries in SCRATCH_POOL. */ static svn_error_t * block_read_changes(apr_array_header_t **changes, svn_fs_t *fs, svn_fs_x__revision_file_t *rev_file, svn_fs_x__p2l_entry_t* entry, + svn_fs_x__changes_context_t *context, svn_boolean_t must_read, apr_pool_t *result_pool, apr_pool_t *scratch_pool) { svn_fs_x__data_t *ffd = fs->fsap_data; svn_stream_t *stream; - svn_revnum_t revision = svn_fs_x__get_revnum(entry->items[0].change_set); - if (!must_read && !ffd->changes_cache) - return SVN_NO_ERROR; + svn_fs_x__pair_cache_key_t key; + svn_fs_x__changes_list_t changes_list; + + /* If we don't have to return any data, just read and cache the first + block. This means we won't cache the remaining blocks from longer + lists right away but only if they are actually needed. */ + apr_size_t next = must_read ? context->next : 0; + apr_size_t next_offset = must_read ? context->next_offset : 0; /* we don't support containers, yet */ SVN_ERR_ASSERT(entry->item_count == 1); + /* The item to read / write. */ + key.revision = svn_fs_x__get_revnum(entry->items[0].change_set); + key.second = next; + /* already in cache? */ - if (!must_read && ffd->changes_cache) + if (!must_read) { svn_boolean_t is_cached = FALSE; - SVN_ERR(svn_cache__has_key(&is_cached, ffd->changes_cache, &revision, + SVN_ERR(svn_cache__has_key(&is_cached, ffd->changes_cache, &key, scratch_pool)); if (is_cached) return SVN_NO_ERROR; } - SVN_ERR(read_item(&stream, fs, rev_file, entry, scratch_pool)); + /* Verify the whole list only once. We don't use the STREAM any further. */ + if (!must_read || next == 0) + SVN_ERR(read_item(&stream, fs, rev_file, entry, scratch_pool)); + + /* Seek to the block to read within the changes list. */ + SVN_ERR(svn_fs_x__rev_file_seek(rev_file, NULL, + entry->offset + next_offset)); + SVN_ERR(svn_fs_x__rev_file_stream(&stream, rev_file)); /* read changes from revision file */ + SVN_ERR(svn_fs_x__read_changes(changes, stream, SVN_FS_X__CHANGES_BLOCK_SIZE, + result_pool, scratch_pool)); - SVN_ERR(svn_fs_x__read_changes(changes, stream, result_pool, scratch_pool)); + SVN_ERR(svn_fs_x__rev_file_offset(&changes_list.end_offset, rev_file)); + changes_list.end_offset -= entry->offset; + changes_list.start_offset = next_offset; + changes_list.count = (*changes)->nelts; + changes_list.changes = (svn_fs_x__change_t **)(*changes)->elts; + changes_list.eol = (changes_list.count < SVN_FS_X__CHANGES_BLOCK_SIZE) + || (changes_list.end_offset + 1 >= entry->size); /* cache for future reference */ - if (ffd->changes_cache) - { - /* Guesstimate for the size of the in-cache representation. */ - apr_size_t estimated_size = (apr_size_t)250 * (*changes)->nelts; + SVN_ERR(svn_cache__set(ffd->changes_cache, &key, &changes_list, + scratch_pool)); - /* Don't even serialize data that probably won't fit into the - * cache. This often implies that either CHANGES is very - * large, memory is scarce or both. Having a huge temporary - * copy would not be a good thing in either case. */ - if (svn_cache__is_cachable(ffd->changes_cache, estimated_size)) - SVN_ERR(svn_cache__set(ffd->changes_cache, &revision, *changes, - scratch_pool)); + /* Trim the result: + * Remove the entries that already been reported. */ + if (must_read) + { + context->next_offset = changes_list.end_offset; + context->eol = changes_list.eol; } return SVN_NO_ERROR; } +/* If not already cached or if MUST_READ is set, read the changed paths + * list container addressed by ENTRY in FS. Return the changes list + * identified by SUB_ITEM in *CHANGES, using CONTEXT to select a sub-range + * within that list. Read the data from REV_FILE and cache the result. + * + * Allocate *CHANGES in RESUSLT_POOL and everything else in SCRATCH_POOL. + */ static svn_error_t * block_read_changes_container(apr_array_header_t **changes, svn_fs_t *fs, svn_fs_x__revision_file_t *rev_file, svn_fs_x__p2l_entry_t* entry, apr_uint32_t sub_item, + svn_fs_x__changes_context_t *context, svn_boolean_t must_read, apr_pool_t *result_pool, apr_pool_t *scratch_pool) @@ -3037,7 +3195,7 @@ block_read_changes_container(apr_array_header_t **changes, key.second = entry->offset; /* already in cache? */ - if (!must_read && ffd->changes_container_cache) + if (!must_read) { svn_boolean_t is_cached = FALSE; SVN_ERR(svn_cache__has_key(&is_cached, ffd->changes_container_cache, @@ -3057,15 +3215,19 @@ block_read_changes_container(apr_array_header_t **changes, if (must_read) SVN_ERR(svn_fs_x__changes_get_list(changes, container, sub_item, - result_pool)); - - if (ffd->changes_container_cache) - SVN_ERR(svn_cache__set(ffd->changes_container_cache, &key, container, - scratch_pool)); + context, result_pool)); + SVN_ERR(svn_cache__set(ffd->changes_container_cache, &key, container, + scratch_pool)); return SVN_NO_ERROR; } +/* If not already cached or if MUST_READ is set, read the node revision + * addressed by ENTRY in FS and return it in *NODEREV_P. Cache the + * result under KEY if caching is enabled. Read the data from REV_FILE. + * Allocate *NODEREV_P in RESUSLT_POOL and allocate temporaries in + * SCRATCH_POOL. + */ static svn_error_t * block_read_noderev(svn_fs_x__noderev_t **noderev_p, svn_fs_t *fs, @@ -3078,14 +3240,12 @@ block_read_noderev(svn_fs_x__noderev_t **noderev_p, { svn_fs_x__data_t *ffd = fs->fsap_data; svn_stream_t *stream; - if (!must_read && !ffd->node_revision_cache) - return SVN_NO_ERROR; /* we don't support containers, yet */ SVN_ERR_ASSERT(entry->item_count == 1); /* already in cache? */ - if (!must_read && ffd->node_revision_cache) + if (!must_read) { svn_boolean_t is_cached = FALSE; SVN_ERR(svn_cache__has_key(&is_cached, ffd->node_revision_cache, key, @@ -3100,13 +3260,18 @@ block_read_noderev(svn_fs_x__noderev_t **noderev_p, SVN_ERR(svn_fs_x__read_noderev(noderev_p, stream, result_pool, scratch_pool)); - if (ffd->node_revision_cache) - SVN_ERR(svn_cache__set(ffd->node_revision_cache, key, *noderev_p, - scratch_pool)); + SVN_ERR(svn_cache__set(ffd->node_revision_cache, key, *noderev_p, + scratch_pool)); return SVN_NO_ERROR; } +/* If not already cached or if MUST_READ is set, read the node revision + * container addressed by ENTRY in FS. Return the item identified by + * SUB_ITEM in *NODEREV_P. Read the data from REV_FILE and cache it. + * Allocate *NODEREV_P in RESUSLT_POOL and allocate temporaries in + * SCRATCH_POOL. + */ static svn_error_t * block_read_noderevs_container(svn_fs_x__noderev_t **noderev_p, svn_fs_t *fs, @@ -3127,7 +3292,7 @@ block_read_noderevs_container(svn_fs_x__noderev_t **noderev_p, key.second = entry->offset; /* already in cache? */ - if (!must_read && ffd->noderevs_container_cache) + if (!must_read) { svn_boolean_t is_cached = FALSE; SVN_ERR(svn_cache__has_key(&is_cached, ffd->noderevs_container_cache, @@ -3147,13 +3312,18 @@ block_read_noderevs_container(svn_fs_x__noderev_t **noderev_p, SVN_ERR(svn_fs_x__noderevs_get(noderev_p, container, sub_item, result_pool)); - if (ffd->noderevs_container_cache) - SVN_ERR(svn_cache__set(ffd->noderevs_container_cache, &key, container, - scratch_pool)); + SVN_ERR(svn_cache__set(ffd->noderevs_container_cache, &key, container, + scratch_pool)); return SVN_NO_ERROR; } +/* If not already cached or if MUST_READ is set, read the representation + * container addressed by ENTRY in FS. Return an extractor object for the + * item identified by SUB_ITEM in *EXTRACTOR. Read the data from REV_FILE + * and cache it. Allocate *EXTRACTOR in RESUSLT_POOL and all temporaries + * in SCRATCH_POOL. + */ static svn_error_t * block_read_reps_container(svn_fs_x__rep_extractor_t **extractor, svn_fs_t *fs, @@ -3174,7 +3344,7 @@ block_read_reps_container(svn_fs_x__rep_extractor_t **extractor, key.second = entry->offset; /* already in cache? */ - if (!must_read && ffd->reps_container_cache) + if (!must_read) { svn_boolean_t is_cached = FALSE; SVN_ERR(svn_cache__has_key(&is_cached, ffd->reps_container_cache, @@ -3195,18 +3365,30 @@ block_read_reps_container(svn_fs_x__rep_extractor_t **extractor, SVN_ERR(svn_fs_x__reps_get(extractor, fs, container, sub_item, result_pool)); - if (ffd->noderevs_container_cache) - SVN_ERR(svn_cache__set(ffd->reps_container_cache, &key, container, - scratch_pool)); + SVN_ERR(svn_cache__set(ffd->reps_container_cache, &key, container, + scratch_pool)); return SVN_NO_ERROR; } +/* Read the whole (e.g. 64kB) block containing the item identified by ID in + * FS and put all data into cache. If necessary and depending on heuristics, + * neighboring blocks may also get read. The data is being read from + * already open REVISION_FILE, which must be the correct rev / pack file + * w.r.t. ID->CHANGE_SET. + * + * For noderevs and changed path lists, the item fetched can be allocated + * RESULT_POOL and returned in *RESULT. Otherwise, RESULT must be NULL. + * The BATON is passed along to the extractor sub-functions and will be + * used only when constructing the *RESULT. SCRATCH_POOL will be used for + * all temporary allocations. + */ static svn_error_t * block_read(void **result, svn_fs_t *fs, const svn_fs_x__id_t *id, svn_fs_x__revision_file_t *revision_file, + void *baton, apr_pool_t *result_pool, apr_pool_t *scratch_pool) { @@ -3232,8 +3414,7 @@ block_read(void **result, do { /* fetch list of items in the block surrounding OFFSET */ - SVN_ERR(aligned_seek(fs, revision_file->file, &block_start, offset, - iterpool)); + SVN_ERR(svn_fs_x__rev_file_seek(revision_file, &block_start, offset)); SVN_ERR(svn_fs_x__p2l_index_lookup(&entries, fs, revision_file, revision, block_start, ffd->block_size, scratch_pool, @@ -3272,8 +3453,8 @@ block_read(void **result, key.revision = svn_fs_x__get_revnum(entry->items[0].change_set); key.second = entry->items[0].number; - SVN_ERR(svn_io_file_seek(revision_file->file, SEEK_SET, - &entry->offset, iterpool)); + SVN_ERR(svn_fs_x__rev_file_seek(revision_file, NULL, + entry->offset)); switch (entry->type) { case SVN_FS_X__ITEM_TYPE_FILE_REP: @@ -3289,17 +3470,16 @@ block_read(void **result, break; case SVN_FS_X__ITEM_TYPE_NODEREV: - if (ffd->node_revision_cache || is_result) - SVN_ERR(block_read_noderev((svn_fs_x__noderev_t **)&item, - fs, revision_file, - entry, &key, is_result, - pool, iterpool)); + SVN_ERR(block_read_noderev((svn_fs_x__noderev_t **)&item, + fs, revision_file, + entry, &key, is_result, + pool, iterpool)); break; case SVN_FS_X__ITEM_TYPE_CHANGES: SVN_ERR(block_read_changes((apr_array_header_t **)&item, fs, revision_file, - entry, is_result, + entry, baton, is_result, pool, iterpool)); break; @@ -3308,7 +3488,8 @@ block_read(void **result, ((apr_array_header_t **)&item, fs, revision_file, entry, wanted_sub_item, - is_result, pool, iterpool)); + baton, is_result, + pool, iterpool)); break; case SVN_FS_X__ITEM_TYPE_NODEREVS_CONT: @@ -3337,7 +3518,7 @@ block_read(void **result, /* if we crossed a block boundary, read the remainder of * the last block as well */ offset = entry->offset + entry->size; - if (offset > block_start + ffd->block_size) + if (offset - block_start > ffd->block_size) ++run_count; svn_pool_clear(iterpool); diff --git a/subversion/libsvn_fs_x/cached_data.h b/subversion/libsvn_fs_x/cached_data.h index 079303eff004..2bcd2d942054 100644 --- a/subversion/libsvn_fs_x/cached_data.h +++ b/subversion/libsvn_fs_x/cached_data.h @@ -20,8 +20,8 @@ * ==================================================================== */ -#ifndef SVN_LIBSVN_FS__CACHED_DATA_H -#define SVN_LIBSVN_FS__CACHED_DATA_H +#ifndef SVN_LIBSVN_FS_X_CACHED_DATA_H +#define SVN_LIBSVN_FS_X_CACHED_DATA_H #include "svn_pools.h" #include "svn_fs.h" @@ -67,7 +67,7 @@ svn_fs_x__rep_chain_length(int *chain_length, svn_fs_t *fs, apr_pool_t *scratch_pool); -/* Set *CONTENTS to be a readable svn_stream_t that receives the text +/* Set *CONTENTS_P to be a readable svn_stream_t that receives the text representation REP as seen in filesystem FS. If CACHE_FULLTEXT is not set, bypass fulltext cache lookup for this rep and don't put the reconstructed fulltext into cache. @@ -79,6 +79,18 @@ svn_fs_x__get_contents(svn_stream_t **contents_p, svn_boolean_t cache_fulltext, apr_pool_t *result_pool); +/* Set *CONTENTS_P to be a readable svn_stream_t that receives the text + representation REP as seen in filesystem FS. Read the latest element + of the delta chain from FILE at offset OFFSET. + Use POOL for allocations. */ +svn_error_t * +svn_fs_x__get_contents_from_file(svn_stream_t **contents_p, + svn_fs_t *fs, + svn_fs_x__representation_t *rep, + apr_file_t *file, + apr_off_t offset, + apr_pool_t *pool); + /* Determine on-disk and expanded sizes of the representation identified * by ENTRY in FS and return the result in PACKED_LEN and EXPANDED_LEN, * respectively. FILE must point to the start of the representation and @@ -168,13 +180,24 @@ svn_fs_x__get_proplist(apr_hash_t **proplist, apr_pool_t *result_pool, apr_pool_t *scratch_pool); -/* Fetch the list of change in revision REV in FS and return it in *CHANGES. - * Allocate the result in POOL. +/* Create a changes retrieval context object in *RESULT_POOL and return it + * in *CONTEXT. It will allow svn_fs_x__get_changes to fetch consecutive + * blocks (one per invocation) from REV's changed paths list in FS. + * Use SCRATCH_POOL for temporary allocations. */ +svn_error_t * +svn_fs_x__create_changes_context(svn_fs_x__changes_context_t **context, + svn_fs_t *fs, + svn_revnum_t rev, + apr_pool_t *result_pool, + apr_pool_t *scratch_pool); + +/* Fetch the block of changes from the CONTEXT and return it in *CHANGES. + * Allocate the result in RESULT_POOL and use SCRATCH_POOL for temporaries. */ svn_error_t * svn_fs_x__get_changes(apr_array_header_t **changes, - svn_fs_t *fs, - svn_revnum_t rev, - apr_pool_t *pool); + svn_fs_x__changes_context_t *context, + apr_pool_t *result_pool, + apr_pool_t *scratch_pool); #endif diff --git a/subversion/libsvn_fs_x/caching.c b/subversion/libsvn_fs_x/caching.c index 17e80bddc698..8c65ab11a2a8 100644 --- a/subversion/libsvn_fs_x/caching.c +++ b/subversion/libsvn_fs_x/caching.c @@ -23,8 +23,7 @@ #include "fs.h" #include "fs_x.h" #include "id.h" -#include "dag.h" -#include "tree.h" +#include "dag_cache.h" #include "index.h" #include "changes.h" #include "noderevs.h" @@ -70,9 +69,9 @@ normalize_key_part(const char *original, return normalized->data; } -/* *CACHE_TXDELTAS, *CACHE_FULLTEXTS and *CACHE_REVPROPS flags will be set - according to FS->CONFIG. *CACHE_NAMESPACE receives the cache prefix - to use. +/* *CACHE_TXDELTAS, *CACHE_FULLTEXTS, *CACHE_REVPROPS and *CACHE_NODEPROPS + flags will be set according to FS->CONFIG. *CACHE_NAMESPACE receives + the cache prefix to use. Allocate CACHE_NAMESPACE in RESULT_POOL. */ static svn_error_t * @@ -80,6 +79,7 @@ read_config(const char **cache_namespace, svn_boolean_t *cache_txdeltas, svn_boolean_t *cache_fulltexts, svn_boolean_t *cache_revprops, + svn_boolean_t *cache_nodeprops, svn_fs_t *fs, apr_pool_t *result_pool) { @@ -138,6 +138,15 @@ read_config(const char **cache_namespace, else *cache_revprops = TRUE; + /* by default, cache nodeprops: this will match pre-1.10 + * behavior where node properties caching was controlled + * by SVN_FS_CONFIG_FSFS_CACHE_FULLTEXTS configuration option. + */ + *cache_nodeprops + = svn_hash__get_bool(fs->config, + SVN_FS_CONFIG_FSFS_CACHE_NODEPROPS, + TRUE); + return SVN_NO_ERROR; } @@ -260,42 +269,42 @@ init_callbacks(svn_cache__t *cache, svn_cache__error_handler_t error_handler, apr_pool_t *pool) { - if (cache != NULL) - { #ifdef SVN_DEBUG_CACHE_DUMP_STATS - /* schedule printing the access statistics upon pool cleanup, - * i.e. end of FSX session. - */ - dump_cache_baton_t *baton; + /* schedule printing the access statistics upon pool cleanup, + * i.e. end of FSX session. + */ + dump_cache_baton_t *baton; - baton = apr_palloc(pool, sizeof(*baton)); - baton->pool = pool; - baton->cache = cache; + baton = apr_palloc(pool, sizeof(*baton)); + baton->pool = pool; + baton->cache = cache; - apr_pool_cleanup_register(pool, - baton, - dump_cache_statistics, - apr_pool_cleanup_null); + apr_pool_cleanup_register(pool, + baton, + dump_cache_statistics, + apr_pool_cleanup_null); #endif - if (error_handler) - SVN_ERR(svn_cache__set_error_handler(cache, - error_handler, - fs, - pool)); - - } + if (error_handler) + SVN_ERR(svn_cache__set_error_handler(cache, + error_handler, + fs, + pool)); return SVN_NO_ERROR; } /* Sets *CACHE_P to cache instance based on provided options. - * Creates memcache if MEMCACHE is not NULL. Creates membuffer cache if - * MEMBUFFER is not NULL. Fallbacks to inprocess cache if MEMCACHE and - * MEMBUFFER are NULL and pages is non-zero. Sets *CACHE_P to NULL - * otherwise. Use the given PRIORITY class for the new cache. If it - * is 0, then use the default priority class. + * + * If DUMMY_CACHE is set, create a null cache. Otherwise, creates a memcache + * if MEMCACHE is not NULL or a membuffer cache if MEMBUFFER is not NULL. + * Falls back to inprocess cache if no other cache type has been selected + * and PAGES is not 0. Create a null cache otherwise. + * + * Use the given PRIORITY class for the new cache. If PRIORITY is 0, then + * use the default priority class. HAS_NAMESPACE indicates whether we + * prefixed this cache instance with a namespace. * * Unless NO_HANDLER is true, register an error handler that reports errors * as warnings to the FS warning callback. @@ -313,8 +322,10 @@ create_cache(svn_cache__t **cache_p, apr_ssize_t klen, const char *prefix, apr_uint32_t priority, + svn_boolean_t has_namespace, svn_fs_t *fs, svn_boolean_t no_handler, + svn_boolean_t dummy_cache, apr_pool_t *result_pool, apr_pool_t *scratch_pool) { @@ -324,7 +335,11 @@ create_cache(svn_cache__t **cache_p, if (priority == 0) priority = SVN_CACHE__MEMBUFFER_DEFAULT_PRIORITY; - if (memcache) + if (dummy_cache) + { + SVN_ERR(svn_cache__create_null(cache_p, prefix, result_pool)); + } + else if (memcache) { SVN_ERR(svn_cache__create_memcache(cache_p, memcache, serializer, deserializer, klen, @@ -335,9 +350,12 @@ create_cache(svn_cache__t **cache_p, } else if (membuffer) { + /* We assume caches with namespaces to be relatively short-lived, + * i.e. their data will not be needed after a while. */ SVN_ERR(svn_cache__create_membuffer_cache( cache_p, membuffer, serializer, deserializer, - klen, prefix, priority, FALSE, result_pool, scratch_pool)); + klen, prefix, priority, FALSE, has_namespace, + result_pool, scratch_pool)); } else if (pages) { @@ -347,7 +365,7 @@ create_cache(svn_cache__t **cache_p, } else { - *cache_p = NULL; + SVN_ERR(svn_cache__create_null(cache_p, prefix, result_pool)); } SVN_ERR(init_callbacks(*cache_p, fs, error_handler, result_pool)); @@ -362,6 +380,7 @@ svn_fs_x__initialize_caches(svn_fs_t *fs, svn_fs_x__data_t *ffd = fs->fsap_data; const char *prefix = apr_pstrcat(scratch_pool, "fsx:", fs->uuid, + "--", ffd->instance_id, "/", normalize_key_part(fs->path, scratch_pool), ":", @@ -371,25 +390,29 @@ svn_fs_x__initialize_caches(svn_fs_t *fs, svn_boolean_t cache_txdeltas; svn_boolean_t cache_fulltexts; svn_boolean_t cache_revprops; + svn_boolean_t cache_nodeprops; const char *cache_namespace; + svn_boolean_t has_namespace; /* Evaluating the cache configuration. */ SVN_ERR(read_config(&cache_namespace, &cache_txdeltas, &cache_fulltexts, &cache_revprops, + &cache_nodeprops, fs, scratch_pool)); prefix = apr_pstrcat(scratch_pool, "ns:", cache_namespace, ":", prefix, SVN_VA_NULL); + has_namespace = strlen(cache_namespace) > 0; membuffer = svn_cache__get_global_membuffer_cache(); /* General rules for assigning cache priorities: * * - Data that can be reconstructed from other elements has low prio - * (e.g. fulltexts, directories etc.) + * (e.g. fulltexts etc.) * - Index data required to find any of the other data has high prio * (e.g. noderevs, L2P and P2L index pages) * - everthing else should use default prio @@ -407,21 +430,6 @@ svn_fs_x__initialize_caches(svn_fs_t *fs, apr_pool_cleanup_null); #endif - /* Rough estimate: revision DAG nodes have size around 320 bytes, so - * let's put 16 on a page. */ - SVN_ERR(create_cache(&(ffd->rev_node_cache), - NULL, - membuffer, - 1024, 16, - svn_fs_x__dag_serialize, - svn_fs_x__dag_deserialize, - APR_HASH_KEY_STRING, - apr_pstrcat(scratch_pool, prefix, "DAG", SVN_VA_NULL), - SVN_CACHE__MEMBUFFER_LOW_PRIORITY, - fs, - no_handler, - fs->pool, scratch_pool)); - /* 1st level DAG node cache */ ffd->dag_node_cache = svn_fs_x__create_dag_cache(fs->pool); @@ -434,25 +442,10 @@ svn_fs_x__initialize_caches(svn_fs_t *fs, svn_fs_x__deserialize_dir_entries, sizeof(svn_fs_x__id_t), apr_pstrcat(scratch_pool, prefix, "DIR", SVN_VA_NULL), - SVN_CACHE__MEMBUFFER_DEFAULT_PRIORITY, - fs, - no_handler, - fs->pool, scratch_pool)); - - /* Only 16 bytes per entry (a revision number + the corresponding offset). - Since we want ~8k pages, that means 512 entries per page. */ - SVN_ERR(create_cache(&(ffd->packed_offset_cache), - NULL, - membuffer, - 32, 1, - svn_fs_x__serialize_manifest, - svn_fs_x__deserialize_manifest, - sizeof(svn_revnum_t), - apr_pstrcat(scratch_pool, prefix, "PACK-MANIFEST", - SVN_VA_NULL), SVN_CACHE__MEMBUFFER_HIGH_PRIORITY, + has_namespace, fs, - no_handler, + no_handler, FALSE, fs->pool, scratch_pool)); /* initialize node revision cache, if caching has been enabled */ @@ -466,8 +459,9 @@ svn_fs_x__initialize_caches(svn_fs_t *fs, apr_pstrcat(scratch_pool, prefix, "NODEREVS", SVN_VA_NULL), SVN_CACHE__MEMBUFFER_HIGH_PRIORITY, + has_namespace, fs, - no_handler, + no_handler, FALSE, fs->pool, scratch_pool)); /* initialize representation header cache, if caching has been enabled */ @@ -481,8 +475,9 @@ svn_fs_x__initialize_caches(svn_fs_t *fs, apr_pstrcat(scratch_pool, prefix, "REPHEADER", SVN_VA_NULL), SVN_CACHE__MEMBUFFER_DEFAULT_PRIORITY, + has_namespace, fs, - no_handler, + no_handler, FALSE, fs->pool, scratch_pool)); /* initialize node change list cache, if caching has been enabled */ @@ -492,140 +487,94 @@ svn_fs_x__initialize_caches(svn_fs_t *fs, 1, 8, /* 1k / entry; 8 entries total, rarely used */ svn_fs_x__serialize_changes, svn_fs_x__deserialize_changes, - sizeof(svn_revnum_t), + sizeof(svn_fs_x__pair_cache_key_t), apr_pstrcat(scratch_pool, prefix, "CHANGES", SVN_VA_NULL), 0, + has_namespace, fs, - no_handler, + no_handler, FALSE, fs->pool, scratch_pool)); /* if enabled, cache fulltext and other derived information */ - if (cache_fulltexts) - { - SVN_ERR(create_cache(&(ffd->fulltext_cache), - ffd->memcache, - membuffer, - 0, 0, /* Do not use inprocess cache */ - /* Values are svn_stringbuf_t */ - NULL, NULL, - sizeof(svn_fs_x__pair_cache_key_t), - apr_pstrcat(scratch_pool, prefix, "TEXT", - SVN_VA_NULL), - SVN_CACHE__MEMBUFFER_DEFAULT_PRIORITY, - fs, - no_handler, - fs->pool, scratch_pool)); - - SVN_ERR(create_cache(&(ffd->properties_cache), - NULL, - membuffer, - 0, 0, /* Do not use inprocess cache */ - svn_fs_x__serialize_properties, - svn_fs_x__deserialize_properties, - sizeof(svn_fs_x__pair_cache_key_t), - apr_pstrcat(scratch_pool, prefix, "PROP", - SVN_VA_NULL), - SVN_CACHE__MEMBUFFER_DEFAULT_PRIORITY, - fs, - no_handler, - fs->pool, scratch_pool)); - - SVN_ERR(create_cache(&(ffd->mergeinfo_cache), - NULL, - membuffer, - 0, 0, /* Do not use inprocess cache */ - svn_fs_x__serialize_mergeinfo, - svn_fs_x__deserialize_mergeinfo, - APR_HASH_KEY_STRING, - apr_pstrcat(scratch_pool, prefix, "MERGEINFO", - SVN_VA_NULL), - 0, - fs, - no_handler, - fs->pool, scratch_pool)); - - SVN_ERR(create_cache(&(ffd->mergeinfo_existence_cache), - NULL, - membuffer, - 0, 0, /* Do not use inprocess cache */ - /* Values are svn_stringbuf_t */ - NULL, NULL, - APR_HASH_KEY_STRING, - apr_pstrcat(scratch_pool, prefix, "HAS_MERGEINFO", - SVN_VA_NULL), - 0, - fs, - no_handler, - fs->pool, scratch_pool)); - } - else - { - ffd->fulltext_cache = NULL; - ffd->properties_cache = NULL; - ffd->mergeinfo_cache = NULL; - ffd->mergeinfo_existence_cache = NULL; - } + SVN_ERR(create_cache(&(ffd->fulltext_cache), + ffd->memcache, + membuffer, + 0, 0, /* Do not use inprocess cache */ + /* Values are svn_stringbuf_t */ + NULL, NULL, + sizeof(svn_fs_x__pair_cache_key_t), + apr_pstrcat(scratch_pool, prefix, "TEXT", + SVN_VA_NULL), + SVN_CACHE__MEMBUFFER_DEFAULT_PRIORITY, + has_namespace, + fs, + no_handler, !cache_fulltexts, + fs->pool, scratch_pool)); + + SVN_ERR(create_cache(&(ffd->properties_cache), + NULL, + membuffer, + 0, 0, /* Do not use inprocess cache */ + svn_fs_x__serialize_properties, + svn_fs_x__deserialize_properties, + sizeof(svn_fs_x__pair_cache_key_t), + apr_pstrcat(scratch_pool, prefix, "PROP", + SVN_VA_NULL), + SVN_CACHE__MEMBUFFER_DEFAULT_PRIORITY, + has_namespace, + fs, + no_handler, !cache_nodeprops, + fs->pool, scratch_pool)); /* if enabled, cache revprops */ - if (cache_revprops) - { - SVN_ERR(create_cache(&(ffd->revprop_cache), - NULL, - membuffer, - 0, 0, /* Do not use inprocess cache */ - svn_fs_x__serialize_properties, - svn_fs_x__deserialize_properties, - sizeof(svn_fs_x__pair_cache_key_t), - apr_pstrcat(scratch_pool, prefix, "REVPROP", - SVN_VA_NULL), - SVN_CACHE__MEMBUFFER_DEFAULT_PRIORITY, - fs, - no_handler, - fs->pool, scratch_pool)); - } - else - { - ffd->revprop_cache = NULL; - } + SVN_ERR(create_cache(&(ffd->revprop_cache), + NULL, + membuffer, + 0, 0, /* Do not use inprocess cache */ + svn_fs_x__serialize_properties, + svn_fs_x__deserialize_properties, + sizeof(svn_fs_x__pair_cache_key_t), + apr_pstrcat(scratch_pool, prefix, "REVPROP", + SVN_VA_NULL), + SVN_CACHE__MEMBUFFER_DEFAULT_PRIORITY, + has_namespace, + fs, + no_handler, !cache_revprops, + fs->pool, scratch_pool)); /* if enabled, cache text deltas and their combinations */ - if (cache_txdeltas) - { - SVN_ERR(create_cache(&(ffd->txdelta_window_cache), - NULL, - membuffer, - 0, 0, /* Do not use inprocess cache */ - svn_fs_x__serialize_txdelta_window, - svn_fs_x__deserialize_txdelta_window, - sizeof(svn_fs_x__window_cache_key_t), - apr_pstrcat(scratch_pool, prefix, "TXDELTA_WINDOW", - SVN_VA_NULL), - SVN_CACHE__MEMBUFFER_LOW_PRIORITY, - fs, - no_handler, - fs->pool, scratch_pool)); - - SVN_ERR(create_cache(&(ffd->combined_window_cache), - NULL, - membuffer, - 0, 0, /* Do not use inprocess cache */ - /* Values are svn_stringbuf_t */ - NULL, NULL, - sizeof(svn_fs_x__window_cache_key_t), - apr_pstrcat(scratch_pool, prefix, "COMBINED_WINDOW", - SVN_VA_NULL), - SVN_CACHE__MEMBUFFER_LOW_PRIORITY, - fs, - no_handler, - fs->pool, scratch_pool)); - } - else - { - ffd->txdelta_window_cache = NULL; - ffd->combined_window_cache = NULL; - } + SVN_ERR(create_cache(&(ffd->txdelta_window_cache), + NULL, + membuffer, + 0, 0, /* Do not use inprocess cache */ + svn_fs_x__serialize_txdelta_window, + svn_fs_x__deserialize_txdelta_window, + sizeof(svn_fs_x__window_cache_key_t), + apr_pstrcat(scratch_pool, prefix, "TXDELTA_WINDOW", + SVN_VA_NULL), + SVN_CACHE__MEMBUFFER_LOW_PRIORITY, + has_namespace, + fs, + no_handler, !cache_txdeltas, + fs->pool, scratch_pool)); + + SVN_ERR(create_cache(&(ffd->combined_window_cache), + NULL, + membuffer, + 0, 0, /* Do not use inprocess cache */ + /* Values are svn_stringbuf_t */ + NULL, NULL, + sizeof(svn_fs_x__window_cache_key_t), + apr_pstrcat(scratch_pool, prefix, "COMBINED_WINDOW", + SVN_VA_NULL), + SVN_CACHE__MEMBUFFER_LOW_PRIORITY, + has_namespace, + fs, + no_handler, !cache_txdeltas, + fs->pool, scratch_pool)); + /* Caches for our various container types. */ SVN_ERR(create_cache(&(ffd->noderevs_container_cache), NULL, membuffer, @@ -636,8 +585,9 @@ svn_fs_x__initialize_caches(svn_fs_t *fs, apr_pstrcat(scratch_pool, prefix, "NODEREVSCNT", SVN_VA_NULL), SVN_CACHE__MEMBUFFER_HIGH_PRIORITY, + has_namespace, fs, - no_handler, + no_handler, FALSE, fs->pool, scratch_pool)); SVN_ERR(create_cache(&(ffd->changes_container_cache), NULL, @@ -649,8 +599,9 @@ svn_fs_x__initialize_caches(svn_fs_t *fs, apr_pstrcat(scratch_pool, prefix, "CHANGESCNT", SVN_VA_NULL), 0, + has_namespace, fs, - no_handler, + no_handler, FALSE, fs->pool, scratch_pool)); SVN_ERR(create_cache(&(ffd->reps_container_cache), NULL, @@ -662,10 +613,12 @@ svn_fs_x__initialize_caches(svn_fs_t *fs, apr_pstrcat(scratch_pool, prefix, "REPSCNT", SVN_VA_NULL), 0, + has_namespace, fs, - no_handler, + no_handler, FALSE, fs->pool, scratch_pool)); + /* Cache index info. */ SVN_ERR(create_cache(&(ffd->l2p_header_cache), NULL, membuffer, @@ -677,8 +630,9 @@ svn_fs_x__initialize_caches(svn_fs_t *fs, apr_pstrcat(scratch_pool, prefix, "L2P_HEADER", SVN_VA_NULL), SVN_CACHE__MEMBUFFER_HIGH_PRIORITY, + has_namespace, fs, - no_handler, + no_handler, FALSE, fs->pool, scratch_pool)); SVN_ERR(create_cache(&(ffd->l2p_page_cache), NULL, @@ -691,8 +645,9 @@ svn_fs_x__initialize_caches(svn_fs_t *fs, apr_pstrcat(scratch_pool, prefix, "L2P_PAGE", SVN_VA_NULL), SVN_CACHE__MEMBUFFER_HIGH_PRIORITY, + has_namespace, fs, - no_handler, + no_handler, FALSE, fs->pool, scratch_pool)); SVN_ERR(create_cache(&(ffd->p2l_header_cache), NULL, @@ -704,8 +659,9 @@ svn_fs_x__initialize_caches(svn_fs_t *fs, apr_pstrcat(scratch_pool, prefix, "P2L_HEADER", SVN_VA_NULL), SVN_CACHE__MEMBUFFER_HIGH_PRIORITY, + has_namespace, fs, - no_handler, + no_handler, FALSE, fs->pool, scratch_pool)); SVN_ERR(create_cache(&(ffd->p2l_page_cache), NULL, @@ -717,8 +673,9 @@ svn_fs_x__initialize_caches(svn_fs_t *fs, apr_pstrcat(scratch_pool, prefix, "P2L_PAGE", SVN_VA_NULL), SVN_CACHE__MEMBUFFER_HIGH_PRIORITY, + has_namespace, fs, - no_handler, + no_handler, FALSE, fs->pool, scratch_pool)); return SVN_NO_ERROR; diff --git a/subversion/libsvn_fs_x/changes.c b/subversion/libsvn_fs_x/changes.c index a7d5ee2fbe52..c385e59da7f6 100644 --- a/subversion/libsvn_fs_x/changes.c +++ b/subversion/libsvn_fs_x/changes.c @@ -21,6 +21,7 @@ */ #include "svn_private_config.h" +#include "svn_sorts.h" #include "private/svn_packed_data.h" @@ -37,8 +38,8 @@ /* the change contains a property modification */ #define CHANGE_PROP_MOD 0x00002 -/* the last part (rev_id) of node revision ID is a transaction ID */ -#define CHANGE_TXN_NODE 0x00004 +/* the change contains a mergeinfo modification */ +#define CHANGE_MERGEINFO_MOD 0x00004 /* (flags & CHANGE_NODE_MASK) >> CHANGE_NODE_SHIFT extracts the node type */ #define CHANGE_NODE_SHIFT 0x00003 @@ -52,16 +53,13 @@ /* (flags & CHANGE_KIND_MASK) >> CHANGE_KIND_SHIFT extracts the change type */ #define CHANGE_KIND_SHIFT 0x00005 -#define CHANGE_KIND_MASK 0x000E0 +#define CHANGE_KIND_MASK 0x00060 /* node types according to svn_fs_path_change_kind_t */ #define CHANGE_KIND_MODIFY 0x00000 #define CHANGE_KIND_ADD 0x00020 #define CHANGE_KIND_DELETE 0x00040 #define CHANGE_KIND_REPLACE 0x00060 -#define CHANGE_KIND_RESET 0x00080 -#define CHANGE_KIND_MOVE 0x000A0 -#define CHANGE_KIND_MOVEREPLACE 0x000C0 /* Our internal representation of a change */ typedef struct binary_change_t @@ -77,10 +75,6 @@ typedef struct binary_change_t svn_revnum_t copyfrom_rev; apr_size_t copyfrom_path; - /* Relevant parts of the node revision ID of the change. - * Empty, if REV_ID is not "used". */ - svn_fs_x__id_t noderev_id; - } binary_change_t; /* The actual container object. Change lists are concatenated into CHANGES @@ -138,20 +132,16 @@ append_change(svn_fs_x__changes_t *changes, svn_fs_x__change_t *change) { binary_change_t binary_change = { 0 }; - svn_boolean_t is_txn_id; /* CHANGE must be sufficiently complete */ SVN_ERR_ASSERT(change); SVN_ERR_ASSERT(change->path.data); - /* Relevant parts of the revision ID of the change. */ - binary_change.noderev_id = change->noderev_id; - /* define the kind of change and what specific information is present */ - is_txn_id = svn_fs_x__is_txn(binary_change.noderev_id.change_set); binary_change.flags = (change->text_mod ? CHANGE_TEXT_MOD : 0) | (change->prop_mod ? CHANGE_PROP_MOD : 0) - | (is_txn_id ? CHANGE_TXN_NODE : 0) + | (change->mergeinfo_mod == svn_tristate_true + ? CHANGE_MERGEINFO_MOD : 0) | ((int)change->change_kind << CHANGE_KIND_SHIFT) | ((int)change->node_kind << CHANGE_NODE_SHIFT); @@ -222,8 +212,11 @@ svn_error_t * svn_fs_x__changes_get_list(apr_array_header_t **list, const svn_fs_x__changes_t *changes, apr_size_t idx, - apr_pool_t *pool) + svn_fs_x__changes_context_t *context, + apr_pool_t *result_pool) { + int list_first; + int list_last; int first; int last; int i; @@ -235,37 +228,46 @@ svn_fs_x__changes_get_list(apr_array_header_t **list, /* validate index */ if (idx + 1 >= (apr_size_t)changes->offsets->nelts) return svn_error_createf(SVN_ERR_FS_CONTAINER_INDEX, NULL, - apr_psprintf(pool, + apr_psprintf(result_pool, _("Changes list index %%%s" " exceeds container size %%d"), APR_SIZE_T_FMT), idx, changes->offsets->nelts - 1); /* range of changes to return */ - first = APR_ARRAY_IDX(changes->offsets, (int)idx, int); - last = APR_ARRAY_IDX(changes->offsets, (int)idx + 1, int); + list_first = APR_ARRAY_IDX(changes->offsets, (int)idx, int); + list_last = APR_ARRAY_IDX(changes->offsets, (int)idx + 1, int); + + /* Restrict it to the sub-range requested by the caller. + * Clip the range to never exceed the list's content. */ + first = MIN(context->next + list_first, list_last); + last = MIN(first + SVN_FS_X__CHANGES_BLOCK_SIZE, list_last); + + /* Indicate to the caller whether the end of the list has been reached. */ + context->eol = last == list_last; /* construct result */ - *list = apr_array_make(pool, last - first, sizeof(svn_fs_x__change_t*)); + *list = apr_array_make(result_pool, last - first, + sizeof(svn_fs_x__change_t*)); for (i = first; i < last; ++i) { const binary_change_t *binary_change = &APR_ARRAY_IDX(changes->changes, i, binary_change_t); /* convert BINARY_CHANGE into a standard FSX svn_fs_x__change_t */ - svn_fs_x__change_t *change = apr_pcalloc(pool, sizeof(*change)); + svn_fs_x__change_t *change = apr_pcalloc(result_pool, sizeof(*change)); change->path.data = svn_fs_x__string_table_get(changes->paths, binary_change->path, &change->path.len, - pool); - - if (binary_change->noderev_id.change_set != SVN_FS_X__INVALID_CHANGE_SET) - change->noderev_id = binary_change->noderev_id; + result_pool); change->change_kind = (svn_fs_path_change_kind_t) ((binary_change->flags & CHANGE_KIND_MASK) >> CHANGE_KIND_SHIFT); change->text_mod = (binary_change->flags & CHANGE_TEXT_MOD) != 0; change->prop_mod = (binary_change->flags & CHANGE_PROP_MOD) != 0; + change->mergeinfo_mod = (binary_change->flags & CHANGE_MERGEINFO_MOD) + ? svn_tristate_true + : svn_tristate_false; change->node_kind = (svn_node_kind_t) ((binary_change->flags & CHANGE_NODE_MASK) >> CHANGE_NODE_SHIFT); @@ -276,7 +278,7 @@ svn_fs_x__changes_get_list(apr_array_header_t **list, = svn_fs_x__string_table_get(changes->paths, binary_change->copyfrom_path, NULL, - pool); + result_pool); /* add it to the result */ APR_ARRAY_PUSH(*list, svn_fs_x__change_t*) = change; @@ -311,8 +313,6 @@ svn_fs_x__write_changes_container(svn_stream_t *stream, svn_packed__create_int_substream(changes_stream, TRUE, FALSE); svn_packed__create_int_substream(changes_stream, TRUE, TRUE); svn_packed__create_int_substream(changes_stream, TRUE, FALSE); - svn_packed__create_int_substream(changes_stream, TRUE, TRUE); - svn_packed__create_int_substream(changes_stream, TRUE, FALSE); /* serialize offsets array */ for (i = 0; i < changes->offsets->nelts; ++i) @@ -330,9 +330,6 @@ svn_fs_x__write_changes_container(svn_stream_t *stream, svn_packed__add_int(changes_stream, change->copyfrom_rev); svn_packed__add_uint(changes_stream, change->copyfrom_path); - - svn_packed__add_int(changes_stream, change->noderev_id.change_set); - svn_packed__add_uint(changes_stream, change->noderev_id.number); } /* write to disk */ @@ -387,9 +384,6 @@ svn_fs_x__read_changes_container(svn_fs_x__changes_t **changes_p, change.copyfrom_rev = (svn_revnum_t)svn_packed__get_int(changes_stream); change.copyfrom_path = (apr_size_t)svn_packed__get_uint(changes_stream); - change.noderev_id.change_set = svn_packed__get_int(changes_stream); - change.noderev_id.number = svn_packed__get_uint(changes_stream); - APR_ARRAY_PUSH(changes->changes, binary_change_t) = change; } @@ -437,14 +431,14 @@ svn_error_t * svn_fs_x__deserialize_changes_container(void **out, void *data, apr_size_t data_len, - apr_pool_t *pool) + apr_pool_t *result_pool) { svn_fs_x__changes_t *changes = (svn_fs_x__changes_t *)data; /* de-serialize sub-structures */ svn_fs_x__deserialize_string_table(changes, &changes->paths); - svn_fs_x__deserialize_apr_array(changes, &changes->changes, pool); - svn_fs_x__deserialize_apr_array(changes, &changes->offsets, pool); + svn_fs_x__deserialize_apr_array(changes, &changes->changes, result_pool); + svn_fs_x__deserialize_apr_array(changes, &changes->offsets, result_pool); /* done */ *out = changes; @@ -464,7 +458,8 @@ svn_fs_x__changes_get_list_func(void **out, int i; apr_array_header_t *list; - apr_uint32_t idx = *(apr_uint32_t *)baton; + svn_fs_x__changes_get_list_baton_t *b = baton; + apr_uint32_t idx = b->sub_item; const svn_fs_x__changes_t *container = data; /* resolve all the sub-container pointers we need */ @@ -495,6 +490,12 @@ svn_fs_x__changes_get_list_func(void **out, first = offsets[idx]; last = offsets[idx+1]; + /* Restrict range to the block requested by the BATON. + * Tell the caller whether we reached the end of the list. */ + first = MIN(first + b->start, last); + last = MIN(first + SVN_FS_X__CHANGES_BLOCK_SIZE, last); + *b->eol = last == offsets[idx+1]; + /* construct result */ list = apr_array_make(pool, last - first, sizeof(svn_fs_x__change_t*)); @@ -508,8 +509,6 @@ svn_fs_x__changes_get_list_func(void **out, = svn_fs_x__string_table_get_func(paths, binary_change->path, &change->path.len, pool); - change->noderev_id = binary_change->noderev_id; - change->change_kind = (svn_fs_path_change_kind_t) ((binary_change->flags & CHANGE_KIND_MASK) >> CHANGE_KIND_SHIFT); change->text_mod = (binary_change->flags & CHANGE_TEXT_MOD) != 0; diff --git a/subversion/libsvn_fs_x/changes.h b/subversion/libsvn_fs_x/changes.h index ccb264761c44..9ba164b3e836 100644 --- a/subversion/libsvn_fs_x/changes.h +++ b/subversion/libsvn_fs_x/changes.h @@ -20,8 +20,8 @@ * ==================================================================== */ -#ifndef SVN_LIBSVN_FS__CHANGES_H -#define SVN_LIBSVN_FS__CHANGES_H +#ifndef SVN_LIBSVN_FS_X_CHANGES_H +#define SVN_LIBSVN_FS_X_CHANGES_H #include "svn_io.h" #include "fs.h" @@ -71,14 +71,16 @@ svn_fs_x__changes_estimate_size(const svn_fs_x__changes_t *changes); /* Read changes containers. */ -/* From CHANGES, extract the change list with the given IDX. Allocate - * the result in POOL and return it in *LIST. +/* From CHANGES, access the change list with the given IDX and extract the + * next entries according to CONTEXT. Allocate the result in RESULT_POOL + * and return it in *LIST. */ svn_error_t * svn_fs_x__changes_get_list(apr_array_header_t **list, const svn_fs_x__changes_t *changes, apr_size_t idx, - apr_pool_t *pool); + svn_fs_x__changes_context_t *context, + apr_pool_t *result_pool); /* I/O interface. */ @@ -114,13 +116,27 @@ svn_error_t * svn_fs_x__deserialize_changes_container(void **out, void *data, apr_size_t data_len, - apr_pool_t *pool); + apr_pool_t *result_pool); + +/* Baton type to be used with svn_fs_x__changes_get_list_func. */ +typedef struct svn_fs_x__changes_get_list_baton_t +{ + /* Sub-item to query */ + apr_uint32_t sub_item; + + /* Deliver data starting from this index within the changes list. */ + int start; + + /* To be set by svn_fs_x__changes_get_list_func: + Did we deliver the last change in that list? */ + svn_boolean_t *eol; +} svn_fs_x__changes_get_list_baton_t; /* Implements svn_cache__partial_getter_func_t for svn_fs_x__changes_t, * setting *OUT to the change list (apr_array_header_t *) selected by - * the apr_uint32_t index passed in as *BATON. This function is similar - * to svn_fs_x__changes_get_list but operates on the cache serialized - * representation of the container. + * the svn_fs_x__changes_get_list_baton_t passed in as *BATON. This + * function is similar to svn_fs_x__changes_get_list but operates on + * the cache serialized representation of the container. */ svn_error_t * svn_fs_x__changes_get_list_func(void **out, diff --git a/subversion/libsvn_fs_x/dag.c b/subversion/libsvn_fs_x/dag.c index 2f5bcb260a6c..5656dbe60034 100644 --- a/subversion/libsvn_fs_x/dag.c +++ b/subversion/libsvn_fs_x/dag.c @@ -1,4 +1,4 @@ -/* dag.c : DAG-like interface filesystem, private to libsvn_fs +/* dag.c : DAG-like interface filesystem * * ==================================================================== * Licensed to the Apache Software Foundation (ASF) under one @@ -41,6 +41,7 @@ #include "svn_private_config.h" #include "private/svn_temp_serializer.h" #include "temp_serializer.h" +#include "dag_cache.h" /* Initializing a filesystem. */ @@ -50,34 +51,12 @@ struct dag_node_t /* The filesystem this dag node came from. */ svn_fs_t *fs; - /* The node revision ID for this dag node. */ - svn_fs_x__id_t id; - - /* In the special case that this node is the root of a transaction - that has not yet been modified, the revision of this node is the - respective txn's base rev. Otherwise, this is SVN_INVALID_REVNUM - for txn nodes and the respective crev for committed nodes. - (Used in svn_fs_node_created_rev.) */ - svn_revnum_t revision; - - /* The node's type (file, dir, etc.) */ - svn_node_kind_t kind; - - /* The node's NODE-REVISION, or NULL if we haven't read it in yet. - This is allocated in this node's POOL. - - If you're willing to respect all the rules above, you can munge - this yourself, but you're probably better off just calling - `get_node_revision' and `set_node_revision', which take care of - things for you. */ + /* The node's NODE-REVISION. */ svn_fs_x__noderev_t *node_revision; /* The pool to allocate NODE_REVISION in. */ apr_pool_t *node_pool; - /* the path at which this node was created. */ - const char *created_path; - /* Directory entry lookup hint to speed up consecutive calls to svn_fs_x__rep_contents_dir_entry(). Only used for directory nodes. Any value is legal but should default to APR_SIZE_MAX. */ @@ -90,20 +69,20 @@ struct dag_node_t svn_node_kind_t svn_fs_x__dag_node_kind(dag_node_t *node) { - return node->kind; + return node->node_revision->kind; } const svn_fs_x__id_t * svn_fs_x__dag_get_id(const dag_node_t *node) { - return &node->id; + return &node->node_revision->noderev_id; } const char * svn_fs_x__dag_get_created_path(dag_node_t *node) { - return node->created_path; + return node->node_revision->created_path; } @@ -144,94 +123,35 @@ copy_node_revision(svn_fs_x__noderev_t *noderev, } -/* Set *NODEREV_P to the cached node-revision for NODE. - If the node-revision was not already cached in NODE, read it in, - allocating the cache in NODE->NODE_POOL. - - If you plan to change the contents of NODE, be careful! We're - handing you a pointer directly to our cached node-revision, not - your own copy. If you change it as part of some operation, but - then some Berkeley DB function deadlocks or gets an error, you'll - need to back out your changes, or else the cache will reflect - changes that never got committed. It's probably best not to change - the structure at all. */ -static svn_error_t * -get_node_revision(svn_fs_x__noderev_t **noderev_p, - dag_node_t *node) -{ - /* If we've already got a copy, there's no need to read it in. */ - if (! node->node_revision) - { - svn_fs_x__noderev_t *noderev; - apr_pool_t *scratch_pool = svn_pool_create(node->node_pool); - - SVN_ERR(svn_fs_x__get_node_revision(&noderev, node->fs, &node->id, - node->node_pool, scratch_pool)); - node->node_revision = noderev; - svn_pool_destroy(scratch_pool); - } - - /* Now NODE->node_revision is set. */ - *noderev_p = node->node_revision; - return SVN_NO_ERROR; -} - -/* Return the node revision ID of NODE. The value returned is shared - with NODE, and will be deallocated when NODE is. */ -svn_error_t * -svn_fs_x__dag_get_node_id(svn_fs_x__id_t *node_id, - dag_node_t *node) +const svn_fs_x__id_t * +svn_fs_x__dag_get_node_id(dag_node_t *node) { - svn_fs_x__noderev_t *noderev; - SVN_ERR(get_node_revision(&noderev, node)); - - *node_id = noderev->node_id; - return SVN_NO_ERROR; + return &node->node_revision->node_id; } -/* Return the node revision ID of NODE. The value returned is shared - with NODE, and will be deallocated when NODE is. */ -svn_error_t * -svn_fs_x__dag_get_copy_id(svn_fs_x__id_t *copy_id, - dag_node_t *node) +const svn_fs_x__id_t * +svn_fs_x__dag_get_copy_id(dag_node_t *node) { - svn_fs_x__noderev_t *noderev; - SVN_ERR(get_node_revision(&noderev, node)); - - *copy_id = noderev->copy_id; - return SVN_NO_ERROR; + return &node->node_revision->copy_id; } -/* Return the node ID of NODE. The value returned is shared with NODE, - and will be deallocated when NODE is. */ -svn_error_t * -svn_fs_x__dag_related_node(svn_boolean_t *same, - dag_node_t *lhs, +svn_boolean_t +svn_fs_x__dag_related_node(dag_node_t *lhs, dag_node_t *rhs) { - svn_fs_x__id_t lhs_node, rhs_node; - - SVN_ERR(svn_fs_x__dag_get_node_id(&lhs_node, lhs)); - SVN_ERR(svn_fs_x__dag_get_node_id(&rhs_node, rhs)); - *same = svn_fs_x__id_eq(&lhs_node, &rhs_node); - - return SVN_NO_ERROR; + return svn_fs_x__id_eq(&lhs->node_revision->node_id, + &rhs->node_revision->node_id); } -svn_error_t * -svn_fs_x__dag_same_line_of_history(svn_boolean_t *same, - dag_node_t *lhs, +svn_boolean_t +svn_fs_x__dag_same_line_of_history(dag_node_t *lhs, dag_node_t *rhs) { - svn_fs_x__noderev_t *lhs_noderev, *rhs_noderev; + svn_fs_x__noderev_t *lhs_noderev = lhs->node_revision; + svn_fs_x__noderev_t *rhs_noderev = rhs->node_revision; - SVN_ERR(get_node_revision(&lhs_noderev, lhs)); - SVN_ERR(get_node_revision(&rhs_noderev, rhs)); - - *same = svn_fs_x__id_eq(&lhs_noderev->node_id, &rhs_noderev->node_id) - && svn_fs_x__id_eq(&lhs_noderev->copy_id, &rhs_noderev->copy_id); - - return SVN_NO_ERROR; + return svn_fs_x__id_eq(&lhs_noderev->node_id, &rhs_noderev->node_id) + && svn_fs_x__id_eq(&lhs_noderev->copy_id, &rhs_noderev->copy_id); } svn_boolean_t @@ -240,7 +160,6 @@ svn_fs_x__dag_check_mutable(const dag_node_t *node) return svn_fs_x__is_txn(svn_fs_x__dag_get_id(node)->change_set); } - svn_error_t * svn_fs_x__dag_get_node(dag_node_t **node, svn_fs_t *fs, @@ -254,7 +173,6 @@ svn_fs_x__dag_get_node(dag_node_t **node, /* Construct the node. */ new_node = apr_pcalloc(result_pool, sizeof(*new_node)); new_node->fs = fs; - new_node->id = *id; new_node->hint = APR_SIZE_MAX; /* Grab the contents so we can inspect the node's kind and created path. */ @@ -263,17 +181,6 @@ svn_fs_x__dag_get_node(dag_node_t **node, new_node->node_pool = result_pool; new_node->node_revision = noderev; - /* Initialize the KIND and CREATED_PATH attributes */ - new_node->kind = noderev->kind; - new_node->created_path = noderev->created_path; - - /* Support our quirky svn_fs_node_created_rev API. - Untouched txn roots report the base rev as theirs. */ - new_node->revision - = ( svn_fs_x__is_fresh_txn_root(noderev) - ? svn_fs_x__get_revnum(noderev->predecessor_id.change_set) - : svn_fs_x__get_revnum(id->change_set)); - /* Return a fresh new node */ *node = new_node; return SVN_NO_ERROR; @@ -283,76 +190,50 @@ svn_fs_x__dag_get_node(dag_node_t **node, svn_revnum_t svn_fs_x__dag_get_revision(const dag_node_t *node) { - return node->revision; + svn_fs_x__noderev_t *noderev = node->node_revision; + return ( svn_fs_x__is_fresh_txn_root(noderev) + ? svn_fs_x__get_revnum(noderev->predecessor_id.change_set) + : svn_fs_x__get_revnum(noderev->noderev_id.change_set)); } - -svn_error_t * -svn_fs_x__dag_get_predecessor_id(svn_fs_x__id_t *id_p, - dag_node_t *node) +const svn_fs_x__id_t * +svn_fs_x__dag_get_predecessor_id(dag_node_t *node) { - svn_fs_x__noderev_t *noderev; - - SVN_ERR(get_node_revision(&noderev, node)); - *id_p = noderev->predecessor_id; - - return SVN_NO_ERROR; + return &node->node_revision->predecessor_id; } - -svn_error_t * -svn_fs_x__dag_get_predecessor_count(int *count, - dag_node_t *node) +int +svn_fs_x__dag_get_predecessor_count(dag_node_t *node) { - svn_fs_x__noderev_t *noderev; - - SVN_ERR(get_node_revision(&noderev, node)); - *count = noderev->predecessor_count; - return SVN_NO_ERROR; + return node->node_revision->predecessor_count; } -svn_error_t * -svn_fs_x__dag_get_mergeinfo_count(apr_int64_t *count, - dag_node_t *node) +apr_int64_t +svn_fs_x__dag_get_mergeinfo_count(dag_node_t *node) { - svn_fs_x__noderev_t *noderev; - - SVN_ERR(get_node_revision(&noderev, node)); - *count = noderev->mergeinfo_count; - return SVN_NO_ERROR; + return node->node_revision->mergeinfo_count; } -svn_error_t * -svn_fs_x__dag_has_mergeinfo(svn_boolean_t *has_mergeinfo, - dag_node_t *node) +svn_boolean_t +svn_fs_x__dag_has_mergeinfo(dag_node_t *node) { - svn_fs_x__noderev_t *noderev; - - SVN_ERR(get_node_revision(&noderev, node)); - *has_mergeinfo = noderev->has_mergeinfo; - return SVN_NO_ERROR; + return node->node_revision->has_mergeinfo; } -svn_error_t * -svn_fs_x__dag_has_descendants_with_mergeinfo(svn_boolean_t *do_they, - dag_node_t *node) +svn_boolean_t +svn_fs_x__dag_has_descendants_with_mergeinfo(dag_node_t *node) { - svn_fs_x__noderev_t *noderev; + svn_fs_x__noderev_t *noderev = node->node_revision; - if (node->kind != svn_node_dir) - { - *do_they = FALSE; - return SVN_NO_ERROR; - } + if (noderev->kind != svn_node_dir) + return FALSE; - SVN_ERR(get_node_revision(&noderev, node)); if (noderev->mergeinfo_count > 1) - *do_they = TRUE; + return TRUE; else if (noderev->mergeinfo_count == 1 && !noderev->has_mergeinfo) - *do_they = TRUE; - else - *do_they = FALSE; - return SVN_NO_ERROR; + return TRUE; + + return FALSE; } @@ -362,16 +243,15 @@ svn_fs_x__dag_has_descendants_with_mergeinfo(svn_boolean_t *do_they, /* Set *ID_P to the noderev-id for entry NAME in PARENT. If no such entry, set *ID_P to NULL but do not error. */ -static svn_error_t * -dir_entry_id_from_node(svn_fs_x__id_t *id_p, +svn_error_t * +svn_fs_x__dir_entry_id(svn_fs_x__id_t *id_p, dag_node_t *parent, const char *name, apr_pool_t *scratch_pool) { svn_fs_x__dirent_t *dirent; - svn_fs_x__noderev_t *noderev; + svn_fs_x__noderev_t *noderev = parent->node_revision; - SVN_ERR(get_node_revision(&noderev, parent)); if (noderev->kind != svn_node_dir) return svn_error_create(SVN_ERR_FS_NOT_DIRECTORY, NULL, _("Can't get entries of non-directory")); @@ -411,14 +291,16 @@ set_entry(dag_node_t *parent, svn_fs_x__txn_id_t txn_id, apr_pool_t *scratch_pool) { - svn_fs_x__noderev_t *parent_noderev; - - /* Get the parent's node-revision. */ - SVN_ERR(get_node_revision(&parent_noderev, parent)); + svn_fs_x__noderev_t *parent_noderev = parent->node_revision; /* Set the new entry. */ - return svn_fs_x__set_entry(parent->fs, txn_id, parent_noderev, name, id, - kind, parent->node_pool, scratch_pool); + SVN_ERR(svn_fs_x__set_entry(parent->fs, txn_id, parent_noderev, name, id, + kind, parent->node_pool, scratch_pool)); + + /* Update cached data. */ + svn_fs_x__update_dag_cache(parent); + + return SVN_NO_ERROR; } @@ -439,7 +321,8 @@ make_entry(dag_node_t **child_p, apr_pool_t *result_pool, apr_pool_t *scratch_pool) { - svn_fs_x__noderev_t new_noderev, *parent_noderev; + svn_fs_x__noderev_t new_noderev; + svn_fs_x__noderev_t *parent_noderev = parent->node_revision; /* Make sure that NAME is a single path component. */ if (! svn_path_is_single_path_component(name)) @@ -448,7 +331,7 @@ make_entry(dag_node_t **child_p, _("Attempted to create a node with an illegal name '%s'"), name); /* Make sure that parent is a directory */ - if (parent->kind != svn_node_dir) + if (parent_noderev->kind != svn_node_dir) return svn_error_create (SVN_ERR_FS_NOT_DIRECTORY, NULL, _("Attempted to create entry in non-directory parent")); @@ -464,7 +347,6 @@ make_entry(dag_node_t **child_p, new_noderev.kind = is_dir ? svn_node_dir : svn_node_file; new_noderev.created_path = svn_fspath__join(parent_path, name, result_pool); - SVN_ERR(get_node_revision(&parent_noderev, parent)); new_noderev.copyroot_path = apr_pstrdup(result_pool, parent_noderev->copyroot_path); new_noderev.copyroot_rev = parent_noderev->copyroot_rev; @@ -495,9 +377,7 @@ svn_fs_x__dag_dir_entries(apr_array_header_t **entries, apr_pool_t *result_pool, apr_pool_t *scratch_pool) { - svn_fs_x__noderev_t *noderev; - - SVN_ERR(get_node_revision(&noderev, node)); + svn_fs_x__noderev_t *noderev = node->node_revision; if (noderev->kind != svn_node_dir) return svn_error_create(SVN_ERR_FS_NOT_DIRECTORY, NULL, @@ -517,7 +397,7 @@ svn_fs_x__dag_set_entry(dag_node_t *node, apr_pool_t *scratch_pool) { /* Check it's a directory. */ - if (node->kind != svn_node_dir) + if (node->node_revision->kind != svn_node_dir) return svn_error_create (SVN_ERR_FS_NOT_DIRECTORY, NULL, _("Attempted to set entry in non-directory node")); @@ -541,16 +421,8 @@ svn_fs_x__dag_get_proplist(apr_hash_t **proplist_p, apr_pool_t *result_pool, apr_pool_t *scratch_pool) { - svn_fs_x__noderev_t *noderev; - apr_hash_t *proplist = NULL; - - SVN_ERR(get_node_revision(&noderev, node)); - - SVN_ERR(svn_fs_x__get_proplist(&proplist, node->fs, noderev, result_pool, - scratch_pool)); - - *proplist_p = proplist; - + SVN_ERR(svn_fs_x__get_proplist(proplist_p, node->fs, node->node_revision, + result_pool, scratch_pool)); return SVN_NO_ERROR; } @@ -560,37 +432,51 @@ svn_fs_x__dag_set_proplist(dag_node_t *node, apr_hash_t *proplist, apr_pool_t *scratch_pool) { - svn_fs_x__noderev_t *noderev; - /* Sanity check: this node better be mutable! */ if (! svn_fs_x__dag_check_mutable(node)) { - svn_string_t *idstr = svn_fs_x__id_unparse(&node->id, scratch_pool); + svn_string_t *idstr + = svn_fs_x__id_unparse(&node->node_revision->noderev_id, + scratch_pool); return svn_error_createf (SVN_ERR_FS_NOT_MUTABLE, NULL, "Can't set proplist on *immutable* node-revision %s", idstr->data); } - /* Go get a fresh NODE-REVISION for this node. */ - SVN_ERR(get_node_revision(&noderev, node)); - /* Set the new proplist. */ - return svn_fs_x__set_proplist(node->fs, noderev, proplist, scratch_pool); + SVN_ERR(svn_fs_x__set_proplist(node->fs, node->node_revision, proplist, + scratch_pool)); + svn_fs_x__update_dag_cache(node); + + return SVN_NO_ERROR; } +/* Write NODE's NODEREV element to disk. Update the DAG cache. + Use SCRATCH_POOL for temporary allocations. */ +static svn_error_t * +noderev_changed(dag_node_t *node, + apr_pool_t *scratch_pool) +{ + SVN_ERR(svn_fs_x__put_node_revision(node->fs, node->node_revision, + scratch_pool)); + svn_fs_x__update_dag_cache(node); + + return SVN_NO_ERROR; +} svn_error_t * svn_fs_x__dag_increment_mergeinfo_count(dag_node_t *node, apr_int64_t increment, apr_pool_t *scratch_pool) { - svn_fs_x__noderev_t *noderev; + svn_fs_x__noderev_t *noderev = node->node_revision; /* Sanity check: this node better be mutable! */ if (! svn_fs_x__dag_check_mutable(node)) { - svn_string_t *idstr = svn_fs_x__id_unparse(&node->id, scratch_pool); + svn_string_t *idstr = svn_fs_x__id_unparse(&noderev->noderev_id, + scratch_pool); return svn_error_createf (SVN_ERR_FS_NOT_MUTABLE, NULL, "Can't increment mergeinfo count on *immutable* node-revision %s", @@ -600,13 +486,11 @@ svn_fs_x__dag_increment_mergeinfo_count(dag_node_t *node, if (increment == 0) return SVN_NO_ERROR; - /* Go get a fresh NODE-REVISION for this node. */ - SVN_ERR(get_node_revision(&noderev, node)); - noderev->mergeinfo_count += increment; if (noderev->mergeinfo_count < 0) { - svn_string_t *idstr = svn_fs_x__id_unparse(&node->id, scratch_pool); + svn_string_t *idstr = svn_fs_x__id_unparse(&noderev->noderev_id, + scratch_pool); return svn_error_createf (SVN_ERR_FS_CORRUPT, NULL, apr_psprintf(scratch_pool, @@ -617,7 +501,8 @@ svn_fs_x__dag_increment_mergeinfo_count(dag_node_t *node, } if (noderev->mergeinfo_count > 1 && noderev->kind == svn_node_file) { - svn_string_t *idstr = svn_fs_x__id_unparse(&node->id, scratch_pool); + svn_string_t *idstr = svn_fs_x__id_unparse(&noderev->noderev_id, + scratch_pool); return svn_error_createf (SVN_ERR_FS_CORRUPT, NULL, apr_psprintf(scratch_pool, @@ -628,7 +513,7 @@ svn_fs_x__dag_increment_mergeinfo_count(dag_node_t *node, } /* Flush it out. */ - return svn_fs_x__put_node_revision(node->fs, noderev, scratch_pool); + return noderev_changed(node, scratch_pool); } svn_error_t * @@ -636,55 +521,38 @@ svn_fs_x__dag_set_has_mergeinfo(dag_node_t *node, svn_boolean_t has_mergeinfo, apr_pool_t *scratch_pool) { - svn_fs_x__noderev_t *noderev; - /* Sanity check: this node better be mutable! */ if (! svn_fs_x__dag_check_mutable(node)) { - svn_string_t *idstr = svn_fs_x__id_unparse(&node->id, scratch_pool); + svn_string_t *idstr + = svn_fs_x__id_unparse(&node->node_revision->noderev_id, + scratch_pool); return svn_error_createf (SVN_ERR_FS_NOT_MUTABLE, NULL, "Can't set mergeinfo flag on *immutable* node-revision %s", idstr->data); } - /* Go get a fresh NODE-REVISION for this node. */ - SVN_ERR(get_node_revision(&noderev, node)); - - noderev->has_mergeinfo = has_mergeinfo; + node->node_revision->has_mergeinfo = has_mergeinfo; /* Flush it out. */ - return svn_fs_x__put_node_revision(node->fs, noderev, scratch_pool); + return noderev_changed(node, scratch_pool); } /*** Roots. ***/ svn_error_t * -svn_fs_x__dag_revision_root(dag_node_t **node_p, - svn_fs_t *fs, - svn_revnum_t rev, - apr_pool_t *result_pool, - apr_pool_t *scratch_pool) -{ - svn_fs_x__id_t root_id; - - svn_fs_x__init_rev_root(&root_id, rev); - return svn_fs_x__dag_get_node(node_p, fs, &root_id, result_pool, - scratch_pool); -} - - -svn_error_t * -svn_fs_x__dag_txn_root(dag_node_t **node_p, - svn_fs_t *fs, - svn_fs_x__txn_id_t txn_id, - apr_pool_t *result_pool, - apr_pool_t *scratch_pool) +svn_fs_x__dag_root(dag_node_t **node_p, + svn_fs_t *fs, + svn_fs_x__change_set_t change_set, + apr_pool_t *result_pool, + apr_pool_t *scratch_pool) { svn_fs_x__id_t root_id; + root_id.change_set = change_set; + root_id.number = SVN_FS_X__ITEM_INDEX_ROOT_NODE; - svn_fs_x__init_txn_root(&root_id, txn_id); return svn_fs_x__dag_get_node(node_p, fs, &root_id, result_pool, scratch_pool); } @@ -734,14 +602,11 @@ svn_fs_x__dag_clone_child(dag_node_t **child_p, } else { - svn_fs_x__noderev_t *noderev, *parent_noderev; - - /* Go get a fresh NODE-REVISION for current child node. */ - SVN_ERR(get_node_revision(&noderev, cur_entry)); + svn_fs_x__noderev_t *noderev = cur_entry->node_revision; if (is_parent_copyroot) { - SVN_ERR(get_node_revision(&parent_noderev, parent)); + svn_fs_x__noderev_t *parent_noderev = parent->node_revision; noderev->copyroot_rev = parent_noderev->copyroot_rev; noderev->copyroot_path = apr_pstrdup(scratch_pool, parent_noderev->copyroot_path); @@ -796,7 +661,7 @@ delete_if_mutable(svn_fs_t *fs, return SVN_NO_ERROR; /* Else it's mutable. Recurse on directories... */ - if (node->kind == svn_node_dir) + if (node->node_revision->kind == svn_node_dir) { apr_array_header_t *entries; int i; @@ -829,13 +694,13 @@ svn_fs_x__dag_delete(dag_node_t *parent, svn_fs_x__txn_id_t txn_id, apr_pool_t *scratch_pool) { - svn_fs_x__noderev_t *parent_noderev; + svn_fs_x__noderev_t *parent_noderev = parent->node_revision; svn_fs_t *fs = parent->fs; svn_fs_x__dirent_t *dirent; apr_pool_t *subpool; /* Make sure parent is a directory. */ - if (parent->kind != svn_node_dir) + if (parent_noderev->kind != svn_node_dir) return svn_error_createf (SVN_ERR_FS_NOT_DIRECTORY, NULL, "Attempted to delete entry '%s' from *non*-directory node", name); @@ -852,9 +717,9 @@ svn_fs_x__dag_delete(dag_node_t *parent, (SVN_ERR_FS_NOT_SINGLE_PATH_COMPONENT, NULL, "Attempted to delete a node with an illegal name '%s'", name); - /* Get a fresh NODE-REVISION for the parent node. */ - SVN_ERR(get_node_revision(&parent_noderev, parent)); - + /* We allocate a few potentially heavy temporary objects (file buffers + and directories). Make sure we don't keep them around for longer + than necessary. */ subpool = svn_pool_create(scratch_pool); /* Search this directory for a dirent with that NAME. */ @@ -871,13 +736,13 @@ svn_fs_x__dag_delete(dag_node_t *parent, "Delete failed--directory has no entry '%s'", name); /* If mutable, remove it and any mutable children from db. */ - SVN_ERR(delete_if_mutable(parent->fs, &dirent->id, scratch_pool)); - svn_pool_destroy(subpool); + SVN_ERR(delete_if_mutable(parent->fs, &dirent->id, subpool)); /* Remove this entry from its parent's entries list. */ - return svn_fs_x__set_entry(parent->fs, txn_id, parent_noderev, name, - NULL, svn_node_unknown, parent->node_pool, - scratch_pool); + SVN_ERR(set_entry(parent, name, NULL, svn_node_unknown, txn_id, subpool)); + + svn_pool_destroy(subpool); + return SVN_NO_ERROR; } @@ -916,23 +781,16 @@ svn_fs_x__dag_get_contents(svn_stream_t **contents_p, dag_node_t *file, apr_pool_t *result_pool) { - svn_fs_x__noderev_t *noderev; - svn_stream_t *contents; - /* Make sure our node is a file. */ - if (file->kind != svn_node_file) + if (file->node_revision->kind != svn_node_file) return svn_error_createf (SVN_ERR_FS_NOT_FILE, NULL, "Attempted to get textual contents of a *non*-file node"); - /* Go get a fresh node-revision for FILE. */ - SVN_ERR(get_node_revision(&noderev, file)); - /* Get a stream to the contents. */ - SVN_ERR(svn_fs_x__get_contents(&contents, file->fs, - noderev->data_rep, TRUE, result_pool)); - - *contents_p = contents; + SVN_ERR(svn_fs_x__get_contents(contents_p, file->fs, + file->node_revision->data_rep, TRUE, + result_pool)); return SVN_NO_ERROR; } @@ -945,23 +803,16 @@ svn_fs_x__dag_get_file_delta_stream(svn_txdelta_stream_t **stream_p, apr_pool_t *result_pool, apr_pool_t *scratch_pool) { - svn_fs_x__noderev_t *src_noderev; - svn_fs_x__noderev_t *tgt_noderev; + svn_fs_x__noderev_t *src_noderev = source ? source->node_revision : NULL; + svn_fs_x__noderev_t *tgt_noderev = target->node_revision; /* Make sure our nodes are files. */ - if ((source && source->kind != svn_node_file) - || target->kind != svn_node_file) + if ((source && src_noderev->kind != svn_node_file) + || tgt_noderev->kind != svn_node_file) return svn_error_createf (SVN_ERR_FS_NOT_FILE, NULL, "Attempted to get textual contents of a *non*-file node"); - /* Go get fresh node-revisions for the nodes. */ - if (source) - SVN_ERR(get_node_revision(&src_noderev, source)); - else - src_noderev = NULL; - SVN_ERR(get_node_revision(&tgt_noderev, target)); - /* Get the delta stream. */ return svn_fs_x__get_file_delta_stream(stream_p, target->fs, src_noderev, tgt_noderev, @@ -976,13 +827,8 @@ svn_fs_x__dag_try_process_file_contents(svn_boolean_t *success, void* baton, apr_pool_t *scratch_pool) { - svn_fs_x__noderev_t *noderev; - - /* Go get fresh node-revisions for the nodes. */ - SVN_ERR(get_node_revision(&noderev, node)); - return svn_fs_x__try_process_file_contents(success, node->fs, - noderev, + node->node_revision, processor, baton, scratch_pool); } @@ -991,18 +837,13 @@ svn_error_t * svn_fs_x__dag_file_length(svn_filesize_t *length, dag_node_t *file) { - svn_fs_x__noderev_t *noderev; - /* Make sure our node is a file. */ - if (file->kind != svn_node_file) + if (file->node_revision->kind != svn_node_file) return svn_error_createf (SVN_ERR_FS_NOT_FILE, NULL, "Attempted to get length of a *non*-file node"); - /* Go get a fresh node-revision for FILE, and . */ - SVN_ERR(get_node_revision(&noderev, file)); - - return svn_fs_x__file_length(length, noderev); + return svn_fs_x__file_length(length, file->node_revision); } @@ -1012,16 +853,13 @@ svn_fs_x__dag_file_checksum(svn_checksum_t **checksum, svn_checksum_kind_t kind, apr_pool_t *result_pool) { - svn_fs_x__noderev_t *noderev; - - if (file->kind != svn_node_file) + if (file->node_revision->kind != svn_node_file) return svn_error_createf (SVN_ERR_FS_NOT_FILE, NULL, "Attempted to get checksum of a *non*-file node"); - SVN_ERR(get_node_revision(&noderev, file)); - - return svn_fs_x__file_checksum(checksum, noderev, kind, result_pool); + return svn_fs_x__file_checksum(checksum, file->node_revision, kind, + result_pool); } @@ -1030,11 +868,8 @@ svn_fs_x__dag_get_edit_stream(svn_stream_t **contents, dag_node_t *file, apr_pool_t *result_pool) { - svn_fs_x__noderev_t *noderev; - svn_stream_t *ws; - /* Make sure our node is a file. */ - if (file->kind != svn_node_file) + if (file->node_revision->kind != svn_node_file) return svn_error_createf (SVN_ERR_FS_NOT_FILE, NULL, "Attempted to set textual contents of a *non*-file node"); @@ -1045,13 +880,8 @@ svn_fs_x__dag_get_edit_stream(svn_stream_t **contents, (SVN_ERR_FS_NOT_MUTABLE, NULL, "Attempted to set textual contents of an immutable node"); - /* Get the node revision. */ - SVN_ERR(get_node_revision(&noderev, file)); - - SVN_ERR(svn_fs_x__set_contents(&ws, file->fs, noderev, result_pool)); - - *contents = ws; - + SVN_ERR(svn_fs_x__set_contents(contents, file->fs, file->node_revision, + result_pool)); return SVN_NO_ERROR; } @@ -1072,9 +902,10 @@ svn_fs_x__dag_finalize_edits(dag_node_t *file, return svn_checksum_mismatch_err(checksum, file_checksum, scratch_pool, _("Checksum mismatch for '%s'"), - file->created_path); + file->node_revision->created_path); } + svn_fs_x__update_dag_cache(file); return SVN_NO_ERROR; } @@ -1086,101 +917,14 @@ svn_fs_x__dag_dup(const dag_node_t *node, /* Allocate our new node. */ dag_node_t *new_node = apr_pmemdup(result_pool, node, sizeof(*new_node)); - /* Only copy cached svn_fs_x__noderev_t for immutable nodes. */ - if (node->node_revision && !svn_fs_x__dag_check_mutable(node)) - { - new_node->node_revision = copy_node_revision(node->node_revision, - result_pool); - new_node->created_path = new_node->node_revision->created_path; - } - else - { - new_node->node_revision = NULL; - new_node->created_path = apr_pstrdup(result_pool, node->created_path); - } - + /* Copy sub-structures. */ + new_node->node_revision = copy_node_revision(node->node_revision, + result_pool); new_node->node_pool = result_pool; return new_node; } -dag_node_t * -svn_fs_x__dag_copy_into_pool(dag_node_t *node, - apr_pool_t *result_pool) -{ - return (node->node_pool == result_pool - ? node - : svn_fs_x__dag_dup(node, result_pool)); -} - -svn_error_t * -svn_fs_x__dag_serialize(void **data, - apr_size_t *data_len, - void *in, - apr_pool_t *pool) -{ - dag_node_t *node = in; - svn_stringbuf_t *serialized; - - /* create an serialization context and serialize the dag node as root */ - svn_temp_serializer__context_t *context = - svn_temp_serializer__init(node, - sizeof(*node), - 1024 - SVN_TEMP_SERIALIZER__OVERHEAD, - pool); - - /* for mutable nodes, we will _never_ cache the noderev */ - if (node->node_revision && !svn_fs_x__dag_check_mutable(node)) - { - svn_fs_x__noderev_serialize(context, &node->node_revision); - } - else - { - svn_temp_serializer__set_null(context, - (const void * const *)&node->node_revision); - svn_temp_serializer__add_string(context, &node->created_path); - } - - /* The deserializer will use its own pool. */ - svn_temp_serializer__set_null(context, - (const void * const *)&node->node_pool); - - /* return serialized data */ - serialized = svn_temp_serializer__get(context); - *data = serialized->data; - *data_len = serialized->len; - - return SVN_NO_ERROR; -} - -svn_error_t * -svn_fs_x__dag_deserialize(void **out, - void *data, - apr_size_t data_len, - apr_pool_t *pool) -{ - dag_node_t *node = (dag_node_t *)data; - if (data_len == 0) - return svn_error_create(SVN_ERR_FS_CORRUPT, NULL, - _("Empty noderev in cache")); - - /* Copy the _full_ buffer as it also contains the sub-structures. */ - node->fs = NULL; - - /* fixup all references to sub-structures */ - svn_fs_x__noderev_deserialize(node, &node->node_revision, pool); - node->node_pool = pool; - - if (node->node_revision) - node->created_path = node->node_revision->created_path; - else - svn_temp_deserializer__resolve(node, (void**)&node->created_path); - - /* return result */ - *out = node; - - return SVN_NO_ERROR; -} svn_error_t * svn_fs_x__dag_open(dag_node_t **child_p, @@ -1192,7 +936,7 @@ svn_fs_x__dag_open(dag_node_t **child_p, svn_fs_x__id_t node_id; /* Ensure that NAME exists in PARENT's entry list. */ - SVN_ERR(dir_entry_id_from_node(&node_id, parent, name, scratch_pool)); + SVN_ERR(svn_fs_x__dir_entry_id(&node_id, parent, name, scratch_pool)); if (! svn_fs_x__id_used(&node_id)) { *child_p = NULL; @@ -1219,13 +963,12 @@ svn_fs_x__dag_copy(dag_node_t *to_node, if (preserve_history) { - svn_fs_x__noderev_t *from_noderev, *to_noderev; + svn_fs_x__noderev_t *to_noderev; svn_fs_x__id_t copy_id; svn_fs_t *fs = svn_fs_x__dag_get_fs(from_node); /* Make a copy of the original node revision. */ - SVN_ERR(get_node_revision(&from_noderev, from_node)); - to_noderev = copy_node_revision(from_noderev, scratch_pool); + to_noderev = copy_node_revision(from_node->node_revision, scratch_pool); /* Reserve a copy ID for this new copy. */ SVN_ERR(svn_fs_x__reserve_copy_id(©_id, fs, txn_id, scratch_pool)); @@ -1253,7 +996,8 @@ svn_fs_x__dag_copy(dag_node_t *to_node, } /* Set the entry in to_node to the new id. */ - return svn_fs_x__dag_set_entry(to_node, entry, id, from_node->kind, + return svn_fs_x__dag_set_entry(to_node, entry, id, + from_node->node_revision->kind, txn_id, scratch_pool); } @@ -1269,7 +1013,8 @@ svn_fs_x__dag_things_different(svn_boolean_t *props_changed, svn_boolean_t strict, apr_pool_t *scratch_pool) { - svn_fs_x__noderev_t *noderev1, *noderev2; + svn_fs_x__noderev_t *noderev1 = node1->node_revision; + svn_fs_x__noderev_t *noderev2 = node2->node_revision; svn_fs_t *fs; svn_boolean_t same; @@ -1280,10 +1025,6 @@ svn_fs_x__dag_things_different(svn_boolean_t *props_changed, fs = svn_fs_x__dag_get_fs(node1); - /* The node revision skels for these two nodes. */ - SVN_ERR(get_node_revision(&noderev1, node1)); - SVN_ERR(get_node_revision(&noderev2, node2)); - /* Compare property keys. */ if (props_changed != NULL) { @@ -1300,48 +1041,25 @@ svn_fs_x__dag_things_different(svn_boolean_t *props_changed, return SVN_NO_ERROR; } -svn_error_t * +void svn_fs_x__dag_get_copyroot(svn_revnum_t *rev, const char **path, dag_node_t *node) { - svn_fs_x__noderev_t *noderev; - - /* Go get a fresh node-revision for NODE. */ - SVN_ERR(get_node_revision(&noderev, node)); - - *rev = noderev->copyroot_rev; - *path = noderev->copyroot_path; - - return SVN_NO_ERROR; + *rev = node->node_revision->copyroot_rev; + *path = node->node_revision->copyroot_path; } -svn_error_t * -svn_fs_x__dag_get_copyfrom_rev(svn_revnum_t *rev, - dag_node_t *node) +svn_revnum_t +svn_fs_x__dag_get_copyfrom_rev(dag_node_t *node) { - svn_fs_x__noderev_t *noderev; - - /* Go get a fresh node-revision for NODE. */ - SVN_ERR(get_node_revision(&noderev, node)); - - *rev = noderev->copyfrom_rev; - - return SVN_NO_ERROR; + return node->node_revision->copyfrom_rev; } -svn_error_t * -svn_fs_x__dag_get_copyfrom_path(const char **path, - dag_node_t *node) +const char * +svn_fs_x__dag_get_copyfrom_path(dag_node_t *node) { - svn_fs_x__noderev_t *noderev; - - /* Go get a fresh node-revision for NODE. */ - SVN_ERR(get_node_revision(&noderev, node)); - - *path = noderev->copyfrom_path; - - return SVN_NO_ERROR; + return node->node_revision->copyfrom_path; } svn_error_t * @@ -1349,20 +1067,17 @@ svn_fs_x__dag_update_ancestry(dag_node_t *target, dag_node_t *source, apr_pool_t *scratch_pool) { - svn_fs_x__noderev_t *source_noderev, *target_noderev; + svn_fs_x__noderev_t *source_noderev = source->node_revision; + svn_fs_x__noderev_t *target_noderev = target->node_revision; if (! svn_fs_x__dag_check_mutable(target)) return svn_error_createf (SVN_ERR_FS_NOT_MUTABLE, NULL, _("Attempted to update ancestry of non-mutable node")); - SVN_ERR(get_node_revision(&source_noderev, source)); - SVN_ERR(get_node_revision(&target_noderev, target)); - target_noderev->predecessor_id = source_noderev->noderev_id; target_noderev->predecessor_count = source_noderev->predecessor_count; target_noderev->predecessor_count++; - return svn_fs_x__put_node_revision(target->fs, target_noderev, - scratch_pool); + return noderev_changed(target, scratch_pool); } diff --git a/subversion/libsvn_fs_x/dag.h b/subversion/libsvn_fs_x/dag.h index 6d5e85baf28d..dd951edcd6f4 100644 --- a/subversion/libsvn_fs_x/dag.h +++ b/subversion/libsvn_fs_x/dag.h @@ -1,4 +1,4 @@ -/* dag.h : DAG-like interface filesystem, private to libsvn_fs +/* dag.h : DAG-like interface filesystem * * ==================================================================== * Licensed to the Apache Software Foundation (ASF) under one @@ -20,8 +20,8 @@ * ==================================================================== */ -#ifndef SVN_LIBSVN_FS_DAG_H -#define SVN_LIBSVN_FS_DAG_H +#ifndef SVN_LIBSVN_FS_X_DAG_H +#define SVN_LIBSVN_FS_X_DAG_H #include "svn_fs.h" #include "svn_delta.h" @@ -82,28 +82,6 @@ dag_node_t * svn_fs_x__dag_dup(const dag_node_t *node, apr_pool_t *result_pool); -/* If NODE has been allocated in POOL, return NODE. Otherwise, return - a copy created in RESULT_POOL with svn_fs_fs__dag_dup. */ -dag_node_t * -svn_fs_x__dag_copy_into_pool(dag_node_t *node, - apr_pool_t *result_pool); - -/* Serialize a DAG node, except don't try to preserve the 'fs' member. - Implements svn_cache__serialize_func_t */ -svn_error_t * -svn_fs_x__dag_serialize(void **data, - apr_size_t *data_len, - void *in, - apr_pool_t *pool); - -/* Deserialize a DAG node, leaving the 'fs' member as NULL. - Implements svn_cache__deserialize_func_t */ -svn_error_t * -svn_fs_x__dag_deserialize(void **out, - void *data, - apr_size_t data_len, - apr_pool_t *pool); - /* Return the filesystem containing NODE. */ svn_fs_t * svn_fs_x__dag_get_fs(dag_node_t *node); @@ -128,27 +106,23 @@ svn_fs_x__dag_get_id(const dag_node_t *node); /* Return the node ID of NODE. The value returned is shared with NODE, and will be deallocated when NODE is. */ -svn_error_t * -svn_fs_x__dag_get_node_id(svn_fs_x__id_t *node_id, - dag_node_t *node); +const svn_fs_x__id_t * +svn_fs_x__dag_get_node_id(dag_node_t *node); /* Return the copy ID of NODE. The value returned is shared with NODE, and will be deallocated when NODE is. */ -svn_error_t * -svn_fs_x__dag_get_copy_id(svn_fs_x__id_t *copy_id, - dag_node_t *node); +const svn_fs_x__id_t * +svn_fs_x__dag_get_copy_id(dag_node_t *node); -/* Set *SAME to TRUE, if nodes LHS and RHS have the same node ID. */ -svn_error_t * -svn_fs_x__dag_related_node(svn_boolean_t *same, - dag_node_t *lhs, +/* Return TRUE, iff nodes LHS and RHS have the same node ID. */ +svn_boolean_t +svn_fs_x__dag_related_node(dag_node_t *lhs, dag_node_t *rhs); -/* Set *SAME to TRUE, if nodes LHS and RHS have the same node and copy IDs. +/* Return TRUE, iff nodes LHS and RHS have the same node and copy IDs. */ -svn_error_t * -svn_fs_x__dag_same_line_of_history(svn_boolean_t *same, - dag_node_t *lhs, +svn_boolean_t +svn_fs_x__dag_same_line_of_history(dag_node_t *lhs, dag_node_t *rhs); /* Return the created path of NODE. The value returned is shared @@ -157,41 +131,31 @@ const char * svn_fs_x__dag_get_created_path(dag_node_t *node); -/* Set *ID_P to the node revision ID of NODE's immediate predecessor. +/* Return the node revision ID of NODE's immediate predecessor. */ -svn_error_t * -svn_fs_x__dag_get_predecessor_id(svn_fs_x__id_t *id_p, - dag_node_t *node); - +const svn_fs_x__id_t * +svn_fs_x__dag_get_predecessor_id(dag_node_t *node); -/* Set *COUNT to the number of predecessors NODE has (recursively). +/* Return the number of predecessors NODE has (recursively). */ -/* ### This function is currently only used by 'verify'. */ -svn_error_t * -svn_fs_x__dag_get_predecessor_count(int *count, - dag_node_t *node); +int +svn_fs_x__dag_get_predecessor_count(dag_node_t *node); -/* Set *COUNT to the number of node under NODE (inclusive) with - svn:mergeinfo properties. +/* Return the number of node under NODE (inclusive) with svn:mergeinfo + properties. */ -svn_error_t * -svn_fs_x__dag_get_mergeinfo_count(apr_int64_t *count, - dag_node_t *node); +apr_int64_t +svn_fs_x__dag_get_mergeinfo_count(dag_node_t *node); -/* Set *DO_THEY to a flag indicating whether or not NODE is a - directory with at least one descendant (not including itself) with - svn:mergeinfo. +/* Return TRUE, iff NODE is a directory with at least one descendant (not + including itself) with svn:mergeinfo. */ -svn_error_t * -svn_fs_x__dag_has_descendants_with_mergeinfo(svn_boolean_t *do_they, - dag_node_t *node); +svn_boolean_t +svn_fs_x__dag_has_descendants_with_mergeinfo(dag_node_t *node); -/* Set *HAS_MERGEINFO to a flag indicating whether or not NODE itself - has svn:mergeinfo set on it. - */ -svn_error_t * -svn_fs_x__dag_has_mergeinfo(svn_boolean_t *has_mergeinfo, - dag_node_t *node); +/* Return TRUE, iff NODE itself has svn:mergeinfo set on it. */ +svn_boolean_t +svn_fs_x__dag_has_mergeinfo(dag_node_t *node); /* Return non-zero IFF NODE is currently mutable. */ svn_boolean_t @@ -252,25 +216,15 @@ svn_fs_x__dag_set_has_mergeinfo(dag_node_t *node, /* Revision and transaction roots. */ -/* Open the root of revision REV of filesystem FS, allocating from +/* Open the root of change set CHANGE_SET of filesystem FS, allocating from RESULT_POOL. Set *NODE_P to the new node. Use SCRATCH_POOL for temporary allocations.*/ svn_error_t * -svn_fs_x__dag_revision_root(dag_node_t **node_p, - svn_fs_t *fs, - svn_revnum_t rev, - apr_pool_t *result_pool, - apr_pool_t *scratch_pool); - - -/* Set *NODE_P to the root of transaction TXN_ID in FS, allocating - from RESULT_POOL. Use SCRATCH_POOL for temporary allocations. */ -svn_error_t * -svn_fs_x__dag_txn_root(dag_node_t **node_p, - svn_fs_t *fs, - svn_fs_x__txn_id_t txn_id, - apr_pool_t *result_pool, - apr_pool_t *scratch_pool); +svn_fs_x__dag_root(dag_node_t **node_p, + svn_fs_t *fs, + svn_fs_x__change_set_t change_set, + apr_pool_t *result_pool, + apr_pool_t *scratch_pool); /* Directories. */ @@ -289,6 +243,14 @@ svn_fs_x__dag_open(dag_node_t **child_p, apr_pool_t *scratch_pool); +/* Set *ID_P to the noderev-id for entry NAME in PARENT. If no such + entry exists, set *ID_P to "unused" but do not error. */ +svn_error_t * +svn_fs_x__dir_entry_id(svn_fs_x__id_t *id_p, + dag_node_t *parent, + const char *name, + apr_pool_t *scratch_pool); + /* Set *ENTRIES_P to an array of NODE's entries, sorted by entry names, and the values are svn_fs_x__dirent_t. The returned table (and elements) is allocated in RESULT_POOL, temporaries in SCRATCH_POOL. */ @@ -548,22 +510,20 @@ svn_fs_x__dag_things_different(svn_boolean_t *props_changed, /* Set *REV and *PATH to the copyroot revision and path of node NODE, or to SVN_INVALID_REVNUM and NULL if no copyroot exists. */ -svn_error_t * +void svn_fs_x__dag_get_copyroot(svn_revnum_t *rev, const char **path, dag_node_t *node); -/* Set *REV to the copyfrom revision associated with NODE. +/* Return the copyfrom revision associated with NODE. */ -svn_error_t * -svn_fs_x__dag_get_copyfrom_rev(svn_revnum_t *rev, - dag_node_t *node); +svn_revnum_t +svn_fs_x__dag_get_copyfrom_rev(dag_node_t *node); -/* Set *PATH to the copyfrom path associated with NODE. +/* Return the copyfrom path associated with NODE. */ -svn_error_t * -svn_fs_x__dag_get_copyfrom_path(const char **path, - dag_node_t *node); +const char * +svn_fs_x__dag_get_copyfrom_path(dag_node_t *node); /* Update *TARGET so that SOURCE is it's predecessor. @@ -577,4 +537,4 @@ svn_fs_x__dag_update_ancestry(dag_node_t *target, } #endif /* __cplusplus */ -#endif /* SVN_LIBSVN_FS_DAG_H */ +#endif /* SVN_LIBSVN_FS_X_DAG_H */ diff --git a/subversion/libsvn_fs_x/dag_cache.c b/subversion/libsvn_fs_x/dag_cache.c new file mode 100644 index 000000000000..896c56fb3203 --- /dev/null +++ b/subversion/libsvn_fs_x/dag_cache.c @@ -0,0 +1,1103 @@ +/* dag_cache.c : DAG walker and node cache. + * + * ==================================================================== + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * ==================================================================== + */ + + +/* The job of this layer is to take a filesystem with lots of node + sharing going on --- the real DAG filesystem as it appears in the + database --- and make it look and act like an ordinary tree + filesystem, with no sharing. + + We do just-in-time cloning: you can walk from some unfinished + transaction's root down into directories and files shared with + committed revisions; as soon as you try to change something, the + appropriate nodes get cloned (and parent directory entries updated) + invisibly, behind your back. Any other references you have to + nodes that have been cloned by other changes, even made by other + processes, are automatically updated to point to the right clones. */ + + +#include <stdlib.h> +#include <string.h> +#include <assert.h> +#include <apr_pools.h> +#include <apr_hash.h> + +#include "svn_hash.h" +#include "svn_private_config.h" +#include "svn_pools.h" +#include "svn_error.h" +#include "svn_path.h" +#include "svn_mergeinfo.h" +#include "svn_fs.h" +#include "svn_props.h" +#include "svn_sorts.h" + +#include "fs.h" +#include "dag.h" +#include "dag_cache.h" +#include "lock.h" +#include "tree.h" +#include "fs_x.h" +#include "fs_id.h" +#include "temp_serializer.h" +#include "cached_data.h" +#include "transaction.h" +#include "pack.h" +#include "util.h" + +#include "private/svn_mergeinfo_private.h" +#include "private/svn_subr_private.h" +#include "private/svn_fs_util.h" +#include "private/svn_fspath.h" +#include "../libsvn_fs/fs-loader.h" + + +/*** Path handling ***/ + +/* DAG caching uses "normalized" paths - which are a relaxed form of + canonical relpaths. They omit the leading '/' of the abspath and trim + any trailing '/'. Any sequences of '//' will be kept as the path walker + simply skips over them. + + Non-canonical sections of the path will therefore only impact efficiency + (extra walker iterations and possibly duplicated entries in the cache) + but not correctness. + + Another optimization is that we don't NUL-terminate the path but strictly + use its length info. That way, it can be traversed easily without + chopping it up and patching it together again. ultimately, however, + the path string is NUL-terminated because we wrapped a NUL-terminated + C string. + */ + +/* Set *RESULT to a normalized version of PATH without actually copying any + string contents. + + For convenience, return the RESULT pointer as the function value too. */ +static svn_string_t * +normalize_path(svn_string_t *result, + const char *path) +{ + apr_size_t len; + + if (path[0] == '/') + ++path; + + len = strlen(path); + while (len && path[len-1] == '/') + --len; + + result->data = path; + result->len = len; + + return result; +} + +/* Extend PATH, i.e. increase its LEN, to cover the next segment. Skip + sequences of '/'. Store the segment in ENTRY and return a pointer to + it C string representation. If no segment has been found (end of path), + return NULL. */ +static const char * +next_entry_name(svn_string_t *path, + svn_stringbuf_t *entry) +{ + const char *segment_start; + const char *segment_end; + + /* Moving to the next segment, skip separators + (normalized does not imply canonical). */ + segment_start = path->data + path->len; + while (*segment_start == '/') + ++segment_start; + + /* End of path? */ + if (*segment_start == '\0') + return NULL; + + /* Find the end of this segment. Note that strchr would not give us the + length of the last segment. */ + segment_end = segment_start; + while (*segment_end != '/' && *segment_end != '\0') + ++segment_end; + + /* Copy the segment into the result buffer. */ + svn_stringbuf_setempty(entry); + svn_stringbuf_appendbytes(entry, segment_start, + segment_end - segment_start); + + /* Extend the "visible" part of the path to the end of that segment. */ + path->len = segment_end - path->data; + + /* Indicate that we found something. */ + return entry->data; +} + +/* Split the normalized PATH into its last segment the corresponding parent + directory. Store them in ENTRY and DIRECTORY, respectively. + + If PATH is empty, return FALSE and produce no output. + Otherwise, return TRUE. + */ +static svn_boolean_t +extract_last_segment(const svn_string_t *path, + svn_string_t *directory, + svn_stringbuf_t *entry) +{ + const char *segment_start; + const char *parent_end; + + /* Edge case. We can't navigate in empty paths. */ + if (path->len == 0) + return FALSE; + + /* Find the start of the last segment. Note that normalized paths never + start nor end with a '/'. */ + segment_start = path->data + path->len - 1; + while (*segment_start != '/' && segment_start != path->data) + --segment_start; + + /* At root level already, i.e. no parent? */ + if (segment_start == path->data) + { + /* Construct result. */ + directory->data = ""; + directory->len = 0; + + svn_stringbuf_setempty(entry); + svn_stringbuf_appendbytes(entry, path->data, path->len); + + return TRUE; + } + + /* Find the end of the parent directory. */ + parent_end = segment_start; + while (parent_end[-1] == '/') + --parent_end; + + /* Construct result. */ + directory->data = path->data; + directory->len = parent_end - path->data; + + ++segment_start; /* previously pointed to the last '/'. */ + svn_stringbuf_setempty(entry); + svn_stringbuf_appendbytes(entry, segment_start, + path->len - (segment_start - path->data)); + + return TRUE; +} + + +/*** Node Caching ***/ + +/* 1st level cache */ + +/* An entry in the first-level cache. REVISION and PATH form the key that + will ultimately be matched. + */ +typedef struct cache_entry_t +{ + /* hash value derived from PATH, REVISION. + Used to short-circuit failed lookups. */ + apr_uint32_t hash_value; + + /* change set to which the NODE belongs */ + svn_fs_x__change_set_t change_set; + + /* path of the NODE */ + char *path; + + /* cached value of strlen(PATH). */ + apr_size_t path_len; + + /* the node allocated in the cache's pool. NULL for empty entries. */ + dag_node_t *node; +} cache_entry_t; + +/* Number of entries in the cache. Keep this low to keep pressure on the + CPU caches low as well. A binary value is most efficient. If we walk + a directory tree, we want enough entries to store nodes for all files + without overwriting the nodes for the parent folder. That way, there + will be no unnecessary misses (except for a few random ones caused by + hash collision). + + The actual number of instances may be higher but entries that got + overwritten are no longer visible. + */ +enum { BUCKET_COUNT = 256 }; + +/* The actual cache structure. All nodes will be allocated in POOL. + When the number of INSERTIONS (i.e. objects created form that pool) + exceeds a certain threshold, the pool will be cleared and the cache + with it. + */ +struct svn_fs_x__dag_cache_t +{ + /* fixed number of (possibly empty) cache entries */ + cache_entry_t buckets[BUCKET_COUNT]; + + /* pool used for all node allocation */ + apr_pool_t *pool; + + /* number of entries created from POOL since the last cleanup */ + apr_size_t insertions; + + /* Property lookups etc. have a very high locality (75% re-hit). + Thus, remember the last hit location for optimistic lookup. */ + apr_size_t last_hit; + + /* Position of the last bucket hit that actually had a DAG node in it. + LAST_HIT may refer to a bucket that matches path@rev but has not + its NODE element set, yet. + This value is a mere hint for optimistic lookup and any value is + valid (as long as it is < BUCKET_COUNT). */ + apr_size_t last_non_empty; +}; + +svn_fs_x__dag_cache_t* +svn_fs_x__create_dag_cache(apr_pool_t *result_pool) +{ + svn_fs_x__dag_cache_t *result = apr_pcalloc(result_pool, sizeof(*result)); + result->pool = svn_pool_create(result_pool); + + return result; +} + +/* Clears the CACHE at regular intervals (destroying all cached nodes). + * Return TRUE if the cache got cleared and previously obtained references + * to cache contents have become invalid. + */ +static svn_boolean_t +auto_clear_dag_cache(svn_fs_x__dag_cache_t* cache) +{ + if (cache->insertions <= BUCKET_COUNT) + return FALSE; + + svn_pool_clear(cache->pool); + + memset(cache->buckets, 0, sizeof(cache->buckets)); + cache->insertions = 0; + + return TRUE; +} + +/* For the given CHANGE_SET and PATH, return the respective entry in CACHE. + If the entry is empty, its NODE member will be NULL and the caller + may then set it to the corresponding DAG node allocated in CACHE->POOL. + */ +static cache_entry_t * +cache_lookup(svn_fs_x__dag_cache_t *cache, + svn_fs_x__change_set_t change_set, + const svn_string_t *path) +{ + apr_size_t i, bucket_index; + apr_size_t path_len = path->len; + apr_uint32_t hash_value = (apr_uint32_t)(apr_uint64_t)change_set; + +#if SVN_UNALIGNED_ACCESS_IS_OK + /* "randomizing" / distributing factor used in our hash function */ + const apr_uint32_t factor = 0xd1f3da69; +#endif + + /* optimistic lookup: hit the same bucket again? */ + cache_entry_t *result = &cache->buckets[cache->last_hit]; + if ( (result->change_set == change_set) + && (result->path_len == path_len) + && !memcmp(result->path, path->data, path_len)) + { + /* Remember the position of the last node we found in this cache. */ + if (result->node) + cache->last_non_empty = cache->last_hit; + + return result; + } + + /* need to do a full lookup. Calculate the hash value + (HASH_VALUE has been initialized to REVISION). */ + i = 0; +#if SVN_UNALIGNED_ACCESS_IS_OK + /* We relax the dependency chain between iterations by processing + two chunks from the input per hash_value self-multiplication. + The HASH_VALUE update latency is now 1 MUL latency + 1 ADD latency + per 2 chunks instead of 1 chunk. + */ + for (; i + 8 <= path_len; i += 8) + hash_value = hash_value * factor * factor + + ( *(const apr_uint32_t*)(path->data + i) * factor + + *(const apr_uint32_t*)(path->data + i + 4)); +#endif + + for (; i < path_len; ++i) + /* Help GCC to minimize the HASH_VALUE update latency by splitting the + MUL 33 of the naive implementation: h = h * 33 + path[i]. This + shortens the dependency chain from 1 shift + 2 ADDs to 1 shift + 1 ADD. + */ + hash_value = hash_value * 32 + (hash_value + (apr_byte_t)path->data[i]); + + bucket_index = hash_value + (hash_value >> 16); + bucket_index = (bucket_index + (bucket_index >> 8)) % BUCKET_COUNT; + + /* access the corresponding bucket and remember its location */ + result = &cache->buckets[bucket_index]; + cache->last_hit = bucket_index; + + /* if it is *NOT* a match, clear the bucket, expect the caller to fill + in the node and count it as an insertion */ + if ( (result->hash_value != hash_value) + || (result->change_set != change_set) + || (result->path_len != path_len) + || memcmp(result->path, path->data, path_len)) + { + result->hash_value = hash_value; + result->change_set = change_set; + + if (result->path_len < path_len || result->path_len == 0) + result->path = apr_palloc(cache->pool, path_len + 1); + result->path_len = path_len; + + memcpy(result->path, path->data, path_len); + result->path[path_len] = 0; + + result->node = NULL; + + cache->insertions++; + } + else if (result->node) + { + /* This bucket is valid & has a suitable DAG node in it. + Remember its location. */ + cache->last_non_empty = bucket_index; + } + + return result; +} + +/* Optimistic lookup using the last seen non-empty location in CACHE. + Return the node of that entry, if it is still in use and matches PATH. + Return NULL otherwise. */ +static dag_node_t * +cache_lookup_last_path(svn_fs_x__dag_cache_t *cache, + const svn_string_t *path) +{ + cache_entry_t *result = &cache->buckets[cache->last_non_empty]; + + if ( result->node + && (result->path_len == path->len) + && !memcmp(result->path, path->data, path->len)) + { + return result->node; + } + + return NULL; +} + +/* Return the cached DAG node for PATH from ROOT's node cache, or NULL if + the node isn't cached. + */ +static dag_node_t * +dag_node_cache_get(svn_fs_root_t *root, + const svn_string_t *path) +{ + svn_fs_x__data_t *ffd = root->fs->fsap_data; + svn_fs_x__change_set_t change_set = svn_fs_x__root_change_set(root); + + auto_clear_dag_cache(ffd->dag_node_cache); + return cache_lookup(ffd->dag_node_cache, change_set, path)->node; +} + + +void +svn_fs_x__update_dag_cache(dag_node_t *node) +{ + svn_fs_x__data_t *ffd = svn_fs_x__dag_get_fs(node)->fsap_data; + const char *path = svn_fs_x__dag_get_created_path(node); + svn_fs_x__dag_cache_t *cache = ffd->dag_node_cache; + + cache_entry_t *bucket; + svn_string_t normalized; + + auto_clear_dag_cache(cache); + bucket = cache_lookup(cache, svn_fs_x__dag_get_id(node)->change_set, + normalize_path(&normalized, path)); + bucket->node = svn_fs_x__dag_dup(node, cache->pool); +} + +void +svn_fs_x__invalidate_dag_cache(svn_fs_root_t *root, + const char *path) +{ + svn_fs_x__data_t *ffd = root->fs->fsap_data; + svn_fs_x__dag_cache_t *cache = ffd->dag_node_cache; + svn_fs_x__change_set_t change_set = svn_fs_x__root_change_set(root); + + apr_size_t i; + for (i = 0; i < BUCKET_COUNT; ++i) + { + cache_entry_t *bucket = &cache->buckets[i]; + if (bucket->change_set == change_set && bucket->node) + { + /* The call to svn_relpath_skip_ancestor() will require both + parameters to be canonical. Since we allow for non-canonical + paths in our cache (unlikely to actually happen), we drop all + such entries. + */ + if (!svn_relpath_is_canonical(bucket->path) + || svn_relpath_skip_ancestor(path + 1, bucket->path)) + bucket->node = NULL; + } + } +} + + +/* Traversing directory paths. */ + +/* Try a short-cut for the open_path() function using the last node accessed. + * If that ROOT is that nodes's "created rev" and PATH matches its "created- + * path", return the node in *NODE_P. Set it to NULL otherwise. + * + * This function is used to support ra_serf-style access patterns where we + * are first asked for path@rev and then for path@c_rev of the same node. + * The shortcut works by ignoring the "rev" part of the cache key and then + * checking whether we got lucky. Lookup and verification are both quick + * plus there are many early outs for common types of mismatch. + */ +static svn_error_t * +try_match_last_node(dag_node_t **node_p, + svn_fs_root_t *root, + const svn_string_t *path) +{ + svn_fs_x__data_t *ffd = root->fs->fsap_data; + + /* Optimistic lookup: if the last node returned from the cache applied to + the same PATH, return it in NODE. */ + dag_node_t *node = cache_lookup_last_path(ffd->dag_node_cache, path); + + /* Did we get a bucket with a committed node? */ + if (node && !svn_fs_x__dag_check_mutable(node)) + { + /* Get the path&rev pair at which this node was created. + This is repository location for which this node is _known_ to be + the right lookup result irrespective of how we found it. */ + const char *created_path + = svn_fs_x__dag_get_created_path(node) + 1; + svn_revnum_t revision = svn_fs_x__dag_get_revision(node); + + /* Is it an exact match? */ + if ( revision == root->rev + && strlen(created_path) == path->len + && memcmp(created_path, path->data, path->len) == 0) + { + svn_fs_x__dag_cache_t *cache = ffd->dag_node_cache; + + /* Insert NODE into the cache at a second location. + In a fraction of all calls, the auto-cleanup may + cause NODE to become invalid. */ + if (!auto_clear_dag_cache(cache)) + { + /* Cache it under its full path@rev access path. */ + svn_fs_x__change_set_t change_set + = svn_fs_x__change_set_by_rev(revision); + cache_entry_t *bucket = cache_lookup(cache, change_set, path); + bucket->node = node; + + *node_p = node; + return SVN_NO_ERROR; + } + } + } + + *node_p = NULL; + return SVN_NO_ERROR; +} + + +/* From directory node PARENT, under ROOT, go one step down to the entry + NAME and return a reference to it in *CHILD_P. + + PATH is the combination of PARENT's path and NAME and is provided by + the caller such that we don't have to construct it here ourselves. + Similarly, CHANGE_SET is redundant with ROOT. + + If the directory entry cannot be found, instead of returning an error, + *CHILD_P will be set to NULL if ALLOW_EMPTY is TRUE. + + NOTE: *NODE_P will live within the DAG cache and we merely return a + reference to it. Hence, it will invalid upon the next cache insertion. + Callers must create a copy if they want a non-temporary object. +*/ +static svn_error_t * +dag_step(dag_node_t **child_p, + svn_fs_root_t *root, + dag_node_t *parent, + const char *name, + const svn_string_t *path, + svn_fs_x__change_set_t change_set, + svn_boolean_t allow_empty, + apr_pool_t *scratch_pool) +{ + svn_fs_t *fs = svn_fs_x__dag_get_fs(parent); + svn_fs_x__data_t *ffd = fs->fsap_data; + cache_entry_t *bucket; + svn_fs_x__id_t node_id; + + /* Locate the corresponding cache entry. We may need PARENT to remain + valid for later use, so don't call auto_clear_dag_cache() here. */ + bucket = cache_lookup(ffd->dag_node_cache, change_set, path); + if (bucket->node) + { + /* Already cached. Return a reference to the cached object. */ + *child_p = bucket->node; + return SVN_NO_ERROR; + } + + /* Get the ID of the node we are looking for. The function call checks + for various error conditions such like PARENT not being a directory. */ + SVN_ERR(svn_fs_x__dir_entry_id(&node_id, parent, name, scratch_pool)); + if (! svn_fs_x__id_used(&node_id)) + { + const char *dir; + + /* No such directory entry. Is a simple NULL result o.k.? */ + if (allow_empty) + { + *child_p = NULL; + return SVN_NO_ERROR; + } + + /* Produce an appropriate error message. */ + dir = apr_pstrmemdup(scratch_pool, path->data, path->len); + dir = svn_fs__canonicalize_abspath(dir, scratch_pool); + + return SVN_FS__NOT_FOUND(root, dir); + } + + /* We are about to add a new entry to the cache. Periodically clear it. + If we had to clear it just now (< 1% chance), re-add the entry for our + item. */ + if (auto_clear_dag_cache(ffd->dag_node_cache)) + bucket = cache_lookup(ffd->dag_node_cache, change_set, path); + + /* Construct the DAG node object for NODE_ID. Let it live in the cache. */ + SVN_ERR(svn_fs_x__dag_get_node(&bucket->node, fs, &node_id, + ffd->dag_node_cache->pool, + scratch_pool)); + + /* Return a reference to the cached object. */ + *child_p = bucket->node; + return SVN_NO_ERROR; +} + +/* Return the CHANGE_SET's root node in *NODE_P. ROOT is the FS API root + object for CHANGE_SET. Use SCRATCH_POOL for temporary allocations. + + NOTE: *NODE_P will live within the DAG cache and we merely return a + reference to it. Hence, it will invalid upon the next cache insertion. + Callers must create a copy if they want a non-temporary object. + */ +static svn_error_t * +get_root_node(dag_node_t **node_p, + svn_fs_root_t *root, + svn_fs_x__change_set_t change_set, + apr_pool_t *scratch_pool) +{ + svn_fs_t *fs = root->fs; + svn_fs_x__data_t *ffd = fs->fsap_data; + cache_entry_t *bucket; + const svn_string_t path = { "", 0 }; + + /* Auto-insert the node in the cache. */ + auto_clear_dag_cache(ffd->dag_node_cache); + bucket = cache_lookup(ffd->dag_node_cache, change_set, &path); + + /* If it is not already cached, construct the DAG node object for NODE_ID. + Let it live in the cache. Sadly, we often can't reuse txn DAG nodes. */ + if (bucket->node == NULL) + SVN_ERR(svn_fs_x__dag_root(&bucket->node, fs, change_set, + ffd->dag_node_cache->pool, scratch_pool)); + + /* Return a reference to the cached object. */ + *node_p = bucket->node; + return SVN_NO_ERROR; +} + +/* Walk the DAG starting at ROOT, following PATH and return a reference to + the target node in *NODE_P. Use SCRATCH_POOL for temporary allocations. + + NOTE: *NODE_P will live within the DAG cache and we merely return a + reference to it. Hence, it will invalid upon the next cache insertion. + Callers must create a copy if they want a non-temporary object. +*/ +static svn_error_t * +walk_dag_path(dag_node_t **node_p, + svn_fs_root_t *root, + svn_string_t *path, + apr_pool_t *scratch_pool) +{ + dag_node_t *here = NULL; /* The directory we're currently looking at. */ + apr_pool_t *iterpool; + svn_fs_x__change_set_t change_set = svn_fs_x__root_change_set(root); + const char *entry; + svn_string_t directory; + svn_stringbuf_t *entry_buffer; + + /* Special case: root directory. + We will later assume that all paths have at least one parent level, + so we must check here for those that don't. */ + if (path->len == 0) + return svn_error_trace(get_root_node(node_p, root, change_set, + scratch_pool)); + + /* Callers often traverse the DAG in some path-based order or along the + history segments. That allows us to try a few guesses about where to + find the next item. This is only useful if the caller didn't request + the full parent chain. */ + + /* First attempt: Assume that we access the DAG for the same path as + in the last lookup but for a different revision that happens to be + the last revision that touched the respective node. This is a + common pattern when e.g. checking out over ra_serf. Note that this + will only work for committed data as the revision info for nodes in + txns is bogus. + + This shortcut is quick and will exit this function upon success. + So, try it first. */ + if (!root->is_txn_root) + { + SVN_ERR(try_match_last_node(node_p, root, path)); + + /* Did the shortcut work? */ + if (*node_p) + return SVN_NO_ERROR; + } + + /* Second attempt: Try starting the lookup immediately at the parent + node. We will often have recently accessed either a sibling or + said parent directory itself for the same revision. ENTRY will + point to the last '/' in PATH. */ + entry_buffer = svn_stringbuf_create_ensure(64, scratch_pool); + if (extract_last_segment(path, &directory, entry_buffer)) + { + here = dag_node_cache_get(root, &directory); + + /* Did the shortcut work? */ + if (here) + return svn_error_trace(dag_step(node_p, root, here, + entry_buffer->data, path, + change_set, FALSE, scratch_pool)); + } + + /* Now there is something to iterate over. Thus, create the ITERPOOL. */ + iterpool = svn_pool_create(scratch_pool); + + /* Make a parent_path item for the root node, using its own current + copy id. */ + SVN_ERR(get_root_node(&here, root, change_set, iterpool)); + path->len = 0; + + /* Walk the path segment by segment. */ + for (entry = next_entry_name(path, entry_buffer); + entry; + entry = next_entry_name(path, entry_buffer)) + { + svn_pool_clear(iterpool); + + /* Note that HERE is allocated from the DAG node cache and will + therefore survive the ITERPOOL cleanup. */ + SVN_ERR(dag_step(&here, root, here, entry, path, change_set, FALSE, + iterpool)); + } + + svn_pool_destroy(iterpool); + *node_p = here; + + return SVN_NO_ERROR; +} + + +/* Return a text string describing the absolute path of parent path + DAG_PATH. It will be allocated in POOL. */ +static const char * +parent_path_path(svn_fs_x__dag_path_t *dag_path, + apr_pool_t *pool) +{ + const char *path_so_far = "/"; + if (dag_path->parent) + path_so_far = parent_path_path(dag_path->parent, pool); + return dag_path->entry + ? svn_fspath__join(path_so_far, dag_path->entry, pool) + : path_so_far; +} + + +/* Choose a copy ID inheritance method *INHERIT_P to be used in the + event that immutable node CHILD in FS needs to be made mutable. If + the inheritance method is copy_id_inherit_new, also return a + *COPY_SRC_PATH on which to base the new copy ID (else return NULL + for that path). CHILD must have a parent (it cannot be the root + node). Temporary allocations are taken from SCRATCH_POOL. */ +static svn_error_t * +get_copy_inheritance(svn_fs_x__copy_id_inherit_t *inherit_p, + const char **copy_src_path, + svn_fs_t *fs, + svn_fs_x__dag_path_t *child, + apr_pool_t *scratch_pool) +{ + svn_fs_x__id_t child_copy_id, parent_copy_id; + const char *id_path = NULL; + svn_fs_root_t *copyroot_root; + dag_node_t *copyroot_node; + svn_revnum_t copyroot_rev; + const char *copyroot_path; + + SVN_ERR_ASSERT(child && child->parent); + + /* Initialize some convenience variables. */ + child_copy_id = *svn_fs_x__dag_get_copy_id(child->node); + parent_copy_id = *svn_fs_x__dag_get_copy_id(child->parent->node); + + /* By default, there is no copy source. */ + *copy_src_path = NULL; + + /* If this child is already mutable, we have nothing to do. */ + if (svn_fs_x__dag_check_mutable(child->node)) + { + *inherit_p = svn_fs_x__copy_id_inherit_self; + return SVN_NO_ERROR; + } + + /* From this point on, we'll assume that the child will just take + its copy ID from its parent. */ + *inherit_p = svn_fs_x__copy_id_inherit_parent; + + /* Special case: if the child's copy ID is '0', use the parent's + copy ID. */ + if (svn_fs_x__id_is_root(&child_copy_id)) + return SVN_NO_ERROR; + + /* Compare the copy IDs of the child and its parent. If they are + the same, then the child is already on the same branch as the + parent, and should use the same mutability copy ID that the + parent will use. */ + if (svn_fs_x__id_eq(&child_copy_id, &parent_copy_id)) + return SVN_NO_ERROR; + + /* If the child is on the same branch that the parent is on, the + child should just use the same copy ID that the parent would use. + Else, the child needs to generate a new copy ID to use should it + need to be made mutable. We will claim that child is on the same + branch as its parent if the child itself is not a branch point, + or if it is a branch point that we are accessing via its original + copy destination path. */ + svn_fs_x__dag_get_copyroot(©root_rev, ©root_path, child->node); + SVN_ERR(svn_fs_x__revision_root(©root_root, fs, copyroot_rev, + scratch_pool)); + SVN_ERR(svn_fs_x__get_temp_dag_node(©root_node, copyroot_root, + copyroot_path, scratch_pool)); + + if (!svn_fs_x__dag_related_node(copyroot_node, child->node)) + return SVN_NO_ERROR; + + /* Determine if we are looking at the child via its original path or + as a subtree item of a copied tree. */ + id_path = svn_fs_x__dag_get_created_path(child->node); + if (strcmp(id_path, parent_path_path(child, scratch_pool)) == 0) + { + *inherit_p = svn_fs_x__copy_id_inherit_self; + return SVN_NO_ERROR; + } + + /* We are pretty sure that the child node is an unedited nested + branched node. When it needs to be made mutable, it should claim + a new copy ID. */ + *inherit_p = svn_fs_x__copy_id_inherit_new; + *copy_src_path = id_path; + return SVN_NO_ERROR; +} + +/* Allocate a new svn_fs_x__dag_path_t node from RESULT_POOL, containing + NODE, ENTRY and PARENT, all copied into RESULT_POOL as well. */ +static svn_fs_x__dag_path_t * +make_parent_path(dag_node_t *node, + const svn_stringbuf_t *entry, + svn_fs_x__dag_path_t *parent, + apr_pool_t *result_pool) +{ + svn_fs_x__dag_path_t *dag_path + = apr_pcalloc(result_pool, sizeof(*dag_path)); + if (node) + dag_path->node = svn_fs_x__dag_dup(node, result_pool); + dag_path->entry = apr_pstrmemdup(result_pool, entry->data, entry->len); + dag_path->parent = parent; + dag_path->copy_inherit = svn_fs_x__copy_id_inherit_unknown; + dag_path->copy_src_path = NULL; + return dag_path; +} + +svn_error_t * +svn_fs_x__get_dag_path(svn_fs_x__dag_path_t **dag_path_p, + svn_fs_root_t *root, + const char *fs_path, + int flags, + svn_boolean_t is_txn_path, + apr_pool_t *result_pool, + apr_pool_t *scratch_pool) +{ + svn_fs_t *fs = root->fs; + dag_node_t *here = NULL; /* The directory we're currently looking at. */ + svn_fs_x__dag_path_t *dag_path; /* The path from HERE up to the root. */ + apr_pool_t *iterpool = svn_pool_create(scratch_pool); + + svn_fs_x__change_set_t change_set = svn_fs_x__root_change_set(root); + const char *entry; + svn_string_t path; + svn_stringbuf_t *entry_buffer = svn_stringbuf_create_ensure(64, + scratch_pool); + apr_size_t path_len; + + /* Normalize the FS_PATH to be compatible with our DAG walk utils. */ + normalize_path(&path, fs_path); /* "" */ + + /* Make a DAG_PATH for the root node, using its own current copy id. */ + SVN_ERR(get_root_node(&here, root, change_set, iterpool)); + dag_path = make_parent_path(here, entry_buffer, NULL, result_pool); + dag_path->copy_inherit = svn_fs_x__copy_id_inherit_self; + + path_len = path.len; + path.len = 0; + + /* Walk the path segment by segment. Add to the DAG_PATH as we go. */ + for (entry = next_entry_name(&path, entry_buffer); + entry; + entry = next_entry_name(&path, entry_buffer)) + { + svn_pool_clear(iterpool); + + /* If the current node is not a directory and we are just here to + * check for the path's existence, then that's o.k. + * Otherwise, non-dir nodes will cause an error in dag_step. */ + if ( (flags & svn_fs_x__dag_path_allow_null) + && (svn_fs_x__dag_node_kind(dag_path->node) != svn_node_dir)) + { + dag_path = NULL; + break; + } + + /* Find the sub-node. */ + SVN_ERR(dag_step(&here, root, dag_path->node, entry, &path, change_set, + TRUE, iterpool)); + + /* "node not found" requires special handling. */ + if (here == NULL) + { + /* If this was the last path component, and the caller + said it was optional, then don't return an error; + just put a NULL node pointer in the path. + */ + if ((flags & svn_fs_x__dag_path_last_optional) + && (path_len == path.len)) + { + dag_path = make_parent_path(NULL, entry_buffer, dag_path, + result_pool); + break; + } + else if (flags & svn_fs_x__dag_path_allow_null) + { + dag_path = NULL; + break; + } + else + { + /* Build a better error message than svn_fs_x__dag_open + can provide, giving the root and full path name. */ + return SVN_FS__NOT_FOUND(root, fs_path); + } + } + + /* Now, make a parent_path item for CHILD. */ + dag_path = make_parent_path(here, entry_buffer, dag_path, result_pool); + if (is_txn_path) + { + SVN_ERR(get_copy_inheritance(&dag_path->copy_inherit, + &dag_path->copy_src_path, + fs, dag_path, iterpool)); + } + } + + svn_pool_destroy(iterpool); + *dag_path_p = dag_path; + return SVN_NO_ERROR; +} + +/* Set *NODE_P to a mutable root directory for ROOT, cloning if + necessary, allocating in RESULT_POOL. ROOT must be a transaction root. + Use ERROR_PATH in error messages. Use SCRATCH_POOL for temporaries.*/ +static svn_error_t * +mutable_root_node(dag_node_t **node_p, + svn_fs_root_t *root, + const char *error_path, + apr_pool_t *result_pool, + apr_pool_t *scratch_pool) +{ + /* If it's not a transaction root, we can't change its contents. */ + if (!root->is_txn_root) + return SVN_FS__ERR_NOT_MUTABLE(root->fs, root->rev, error_path); + + /* It's a transaction root. + Get the appropriate DAG root node and copy it into RESULT_POOL. */ + SVN_ERR(get_root_node(node_p, root, svn_fs_x__root_change_set(root), + scratch_pool)); + *node_p = svn_fs_x__dag_dup(*node_p, result_pool); + + return SVN_NO_ERROR; +} + +svn_error_t * +svn_fs_x__make_path_mutable(svn_fs_root_t *root, + svn_fs_x__dag_path_t *parent_path, + const char *error_path, + apr_pool_t *result_pool, + apr_pool_t *scratch_pool) +{ + dag_node_t *clone; + svn_fs_x__txn_id_t txn_id = svn_fs_x__root_txn_id(root); + + /* Is the node mutable already? */ + if (svn_fs_x__dag_check_mutable(parent_path->node)) + return SVN_NO_ERROR; + + /* Are we trying to clone the root, or somebody's child node? */ + if (parent_path->parent) + { + svn_fs_x__id_t copy_id = { SVN_INVALID_REVNUM, 0 }; + svn_fs_x__id_t *copy_id_ptr = ©_id; + svn_fs_x__copy_id_inherit_t inherit = parent_path->copy_inherit; + const char *clone_path, *copyroot_path; + svn_revnum_t copyroot_rev; + svn_boolean_t is_parent_copyroot = FALSE; + svn_fs_root_t *copyroot_root; + dag_node_t *copyroot_node; + apr_pool_t *subpool; + + /* We're trying to clone somebody's child. Make sure our parent + is mutable. */ + SVN_ERR(svn_fs_x__make_path_mutable(root, parent_path->parent, + error_path, result_pool, + scratch_pool)); + + /* Allocate all temporaries in a sub-pool that we control locally. + That way, we keep only the data of one level of recursion around + at any time. */ + subpool = svn_pool_create(scratch_pool); + switch (inherit) + { + case svn_fs_x__copy_id_inherit_parent: + copy_id = *svn_fs_x__dag_get_copy_id(parent_path->parent->node); + break; + + case svn_fs_x__copy_id_inherit_new: + SVN_ERR(svn_fs_x__reserve_copy_id(©_id, root->fs, txn_id, + subpool)); + break; + + case svn_fs_x__copy_id_inherit_self: + copy_id_ptr = NULL; + break; + + case svn_fs_x__copy_id_inherit_unknown: + default: + SVN_ERR_MALFUNCTION(); /* uh-oh -- somebody didn't calculate copy-ID + inheritance data. */ + } + + /* Determine what copyroot our new child node should use. */ + svn_fs_x__dag_get_copyroot(©root_rev, ©root_path, + parent_path->node); + SVN_ERR(svn_fs_x__revision_root(©root_root, root->fs, + copyroot_rev, subpool)); + SVN_ERR(svn_fs_x__get_temp_dag_node(©root_node, copyroot_root, + copyroot_path, subpool)); + + if (!svn_fs_x__dag_related_node(copyroot_node, parent_path->node)) + is_parent_copyroot = TRUE; + + /* Now make this node mutable. */ + clone_path = parent_path_path(parent_path->parent, subpool); + SVN_ERR(svn_fs_x__dag_clone_child(&clone, + parent_path->parent->node, + clone_path, + parent_path->entry, + copy_id_ptr, txn_id, + is_parent_copyroot, + result_pool, + subpool)); + + /* Update the path cache. */ + svn_fs_x__update_dag_cache(clone); + svn_pool_destroy(subpool); + } + else + { + /* We're trying to clone the root directory. */ + SVN_ERR(mutable_root_node(&clone, root, error_path, result_pool, + scratch_pool)); + } + + /* Update the PARENT_PATH link to refer to the clone. */ + parent_path->node = clone; + + return SVN_NO_ERROR; +} + + +svn_error_t * +svn_fs_x__get_temp_dag_node(dag_node_t **node_p, + svn_fs_root_t *root, + const char *path, + apr_pool_t *scratch_pool) +{ + svn_string_t normalized; + + /* First we look for the DAG in our cache. */ + *node_p = dag_node_cache_get(root, normalize_path(&normalized, path)); + + /* If it is not there, walk the DAG and fill the cache. */ + if (! *node_p) + SVN_ERR(walk_dag_path(node_p, root, &normalized, scratch_pool)); + + return SVN_NO_ERROR; +} + + +svn_error_t * +svn_fs_x__get_dag_node(dag_node_t **dag_node_p, + svn_fs_root_t *root, + const char *path, + apr_pool_t *result_pool, + apr_pool_t *scratch_pool) +{ + dag_node_t *node = NULL; + SVN_ERR(svn_fs_x__get_temp_dag_node(&node, root, path, scratch_pool)); + + /* We want the returned node to live in POOL. */ + *dag_node_p = svn_fs_x__dag_dup(node, result_pool); + + return SVN_NO_ERROR; +} diff --git a/subversion/libsvn_fs_x/dag_cache.h b/subversion/libsvn_fs_x/dag_cache.h new file mode 100644 index 000000000000..fd78bb74f9e8 --- /dev/null +++ b/subversion/libsvn_fs_x/dag_cache.h @@ -0,0 +1,174 @@ +/* dag_cache.h : Interface to the DAG walker and node cache. + * + * ==================================================================== + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * ==================================================================== + */ + +#ifndef SVN_LIBSVN_FS_X_DAG_CACHE_H +#define SVN_LIBSVN_FS_X_DAG_CACHE_H + +#include "dag.h" + +#ifdef __cplusplus +extern "C" { +#endif /* __cplusplus */ + +/* In RESULT_POOL, create an instance of a DAG node cache. */ +svn_fs_x__dag_cache_t* +svn_fs_x__create_dag_cache(apr_pool_t *result_pool); + +/* Invalidate cache entries for PATH within ROOT and any of its children. */ +void +svn_fs_x__invalidate_dag_cache(svn_fs_root_t *root, + const char *path); + +/* Flag type used in svn_fs_x__dag_path_t to determine where the + respective node got its copy ID from. */ +typedef enum svn_fs_x__copy_id_inherit_t +{ + svn_fs_x__copy_id_inherit_unknown = 0, + svn_fs_x__copy_id_inherit_self, + svn_fs_x__copy_id_inherit_parent, + svn_fs_x__copy_id_inherit_new + +} svn_fs_x__copy_id_inherit_t; + +/* Flags for svn_fs_x__get_dag_path. */ +typedef enum svn_fs_x__dag_path_flags_t { + + /* The last component of the PATH need not exist. (All parent + directories must exist, as usual.) If the last component doesn't + exist, simply leave the `node' member of the bottom parent_path + component zero. */ + svn_fs_x__dag_path_last_optional = 1, + + /* The caller wants a NULL path object instead of an error if the + path cannot be found. */ + svn_fs_x__dag_path_allow_null = 2 +} svn_fs_x__dag_path_flags_t; + + +/* A linked list representing the path from a node up to a root + directory. We use this for cloning, and for operations that need + to deal with both a node and its parent directory. For example, a + `delete' operation needs to know that the node actually exists, but + also needs to change the parent directory. */ +typedef struct svn_fs_x__dag_path_t +{ + /* A node along the path. This could be the final node, one of its + parents, or the root. Every parent path ends with an element for + the root directory. */ + dag_node_t *node; + + /* The name NODE has in its parent directory. This is zero for the + root directory, which (obviously) has no name in its parent. */ + char *entry; + + /* The parent of NODE, or zero if NODE is the root directory. */ + struct svn_fs_x__dag_path_t *parent; + + /* The copy ID inheritance style. */ + svn_fs_x__copy_id_inherit_t copy_inherit; + + /* If copy ID inheritance style is copy_id_inherit_new, this is the + path which should be implicitly copied; otherwise, this is NULL. */ + const char *copy_src_path; + +} svn_fs_x__dag_path_t; + +/* Open the node identified by PATH in ROOT, allocating in RESULT_POOL. + Set *DAG_PATH_P to a path from the node up to ROOT. The resulting + **DAG_PATH_P value is guaranteed to contain at least one element, + for the root directory. PATH must be in canonical form. Allocate + temporaries from SCRATCH_POOL. + + If resulting *PARENT_PATH_P will eventually be made mutable and + modified, or if copy ID inheritance information is otherwise needed, + IS_TXN_PATH must be set. If IS_TXN_PATH is FALSE, no copy ID + inheritance information will be calculated for the *PARENT_PATH_P chain. + + If FLAGS & open_path_last_optional is zero, return the error + SVN_ERR_FS_NOT_FOUND if the node PATH refers to does not exist. If + non-zero, require all the parent directories to exist as normal, + but if the final path component doesn't exist, simply return a path + whose bottom `node' member is zero. This option is useful for + callers that create new nodes --- we find the parent directory for + them, and tell them whether the entry exists already. + + If FLAGS & svn_fs_x__dag_path_allow_null is non-zero, set the + *PARENT_PATH_P to NULL if any node in the path could not be found. + + NOTE: Public interfaces which only *read* from the filesystem + should not call this function directly, but should instead use + svn_fs_x__get_dag_node(). +*/ +svn_error_t * +svn_fs_x__get_dag_path(svn_fs_x__dag_path_t **dag_path_p, + svn_fs_root_t *root, + const char *path, + int flags, + svn_boolean_t is_txn_path, + apr_pool_t *result_pool, + apr_pool_t *scratch_pool); + +/* Make the node referred to by PARENT_PATH mutable, if it isn't already, + allocating from RESULT_POOL. ROOT must be the root from which + PARENT_PATH descends. Clone any parent directories as needed. + Adjust the dag nodes in PARENT_PATH to refer to the clones. Use + ERROR_PATH in error messages. Use SCRATCH_POOL for temporaries. */ +svn_error_t * +svn_fs_x__make_path_mutable(svn_fs_root_t *root, + svn_fs_x__dag_path_t *parent_path, + const char *error_path, + apr_pool_t *result_pool, + apr_pool_t *scratch_pool); + +/* Open the node identified by PATH in ROOT. Set *DAG_NODE_P to a temporary + reference to the node we find. That reference will become invalid upon + the next access to the DAG cache. + + Return the error SVN_ERR_FS_NOT_FOUND if this node doesn't exist. + */ +svn_error_t * +svn_fs_x__get_temp_dag_node(dag_node_t **node_p, + svn_fs_root_t *root, + const char *path, + apr_pool_t *scratch_pool); + +/* Open the node identified by PATH in ROOT. Set *DAG_NODE_P to the + node we find, allocated in RESULT_POOL. Return the error + SVN_ERR_FS_NOT_FOUND if this node doesn't exist. Use SCRATCH_POOL + for temporary allocations. + */ +svn_error_t * +svn_fs_x__get_dag_node(dag_node_t **dag_node_p, + svn_fs_root_t *root, + const char *path, + apr_pool_t *result_pool, + apr_pool_t *scratch_pool); + +/* Add / update the NODE in the node cache. */ +void +svn_fs_x__update_dag_cache(dag_node_t *node); + +#ifdef __cplusplus +} +#endif /* __cplusplus */ + +#endif /* SVN_LIBSVN_FS_X_DAG_CACHE_H */ diff --git a/subversion/libsvn_fs_x/fs.c b/subversion/libsvn_fs_x/fs.c index abc564db34dc..7eb0add2def0 100644 --- a/subversion/libsvn_fs_x/fs.c +++ b/subversion/libsvn_fs_x/fs.c @@ -27,12 +27,12 @@ #include <apr_general.h> #include <apr_pools.h> #include <apr_file_io.h> -#include <apr_thread_mutex.h> #include "svn_fs.h" #include "svn_delta.h" #include "svn_version.h" #include "svn_pools.h" +#include "batch_fsync.h" #include "fs.h" #include "fs_x.h" #include "pack.h" @@ -137,6 +137,18 @@ x_serialized_init(svn_fs_t *fs, return SVN_NO_ERROR; } +svn_error_t * +svn_fs_x__initialize_shared_data(svn_fs_t *fs, + svn_mutex__t *common_pool_lock, + apr_pool_t *scratch_pool, + apr_pool_t *common_pool) +{ + SVN_MUTEX__WITH_LOCK(common_pool_lock, + x_serialized_init(fs, common_pool, scratch_pool)); + + return SVN_NO_ERROR; +} + /* This function is provided for Subversion 1.0.x compatibility. It @@ -218,20 +230,11 @@ x_info(const void **fsx_info, return SVN_NO_ERROR; } -/* Wrapper around svn_fs_x__revision_prop() adapting between function - signatures. */ static svn_error_t * -x_revision_prop(svn_string_t **value_p, - svn_fs_t *fs, - svn_revnum_t rev, - const char *propname, - apr_pool_t *pool) +x_refresh_revprops(svn_fs_t *fs, + apr_pool_t *scratch_pool) { - apr_pool_t *scratch_pool = svn_pool_create(pool); - SVN_ERR(svn_fs_x__revision_prop(value_p, fs, rev, propname, pool, - scratch_pool)); - svn_pool_destroy(scratch_pool); - + svn_fs_x__invalidate_revprop_generation(fs); return SVN_NO_ERROR; } @@ -241,14 +244,14 @@ static svn_error_t * x_revision_proplist(apr_hash_t **proplist_p, svn_fs_t *fs, svn_revnum_t rev, - apr_pool_t *pool) + svn_boolean_t refresh, + apr_pool_t *result_pool, + apr_pool_t *scratch_pool) { - apr_pool_t *scratch_pool = svn_pool_create(pool); - /* No need to bypass the caches for r/o access to revprops. */ SVN_ERR(svn_fs_x__get_revision_proplist(proplist_p, fs, rev, FALSE, - pool, scratch_pool)); - svn_pool_destroy(scratch_pool); + refresh, result_pool, + scratch_pool)); return SVN_NO_ERROR; } @@ -262,7 +265,8 @@ x_set_uuid(svn_fs_t *fs, { /* Whenever we set a new UUID, imply that FS will also be a different * instance (on formats that support this). */ - return svn_error_trace(svn_fs_x__set_uuid(fs, uuid, NULL, scratch_pool)); + return svn_error_trace(svn_fs_x__set_uuid(fs, uuid, NULL, TRUE, + scratch_pool)); } /* Wrapper around svn_fs_x__begin_txn() providing the scratch pool. */ @@ -285,7 +289,8 @@ x_begin_txn(svn_fs_txn_t **txn_p, /* The vtable associated with a specific open filesystem. */ static fs_vtable_t fs_vtable = { svn_fs_x__youngest_rev, - x_revision_prop, + x_refresh_revprops, + svn_fs_x__revision_prop, x_revision_proplist, svn_fs_x__change_rev_prop, x_set_uuid, @@ -316,6 +321,9 @@ static svn_error_t * initialize_fs_struct(svn_fs_t *fs) { svn_fs_x__data_t *ffd = apr_pcalloc(fs->pool, sizeof(*ffd)); + ffd->revprop_generation = -1; + ffd->flush_to_disk = TRUE; + fs->vtable = &fs_vtable; fs->fsap_data = ffd; return SVN_NO_ERROR; @@ -500,7 +508,7 @@ x_pack(svn_fs_t *fs, apr_pool_t *common_pool) { SVN_ERR(x_open(fs, path, common_pool_lock, scratch_pool, common_pool)); - return svn_fs_x__pack(fs, notify_func, notify_baton, + return svn_fs_x__pack(fs, 0, notify_func, notify_baton, cancel_func, cancel_baton, scratch_pool); } @@ -536,24 +544,17 @@ x_hotcopy(svn_fs_t *src_fs, if (cancel_func) SVN_ERR(cancel_func(cancel_baton)); - /* Test target repo when in INCREMENTAL mode, initialize it when not. - * For this, we need our FS internal data structures to be temporarily - * available. */ + SVN_ERR(svn_fs__check_fs(dst_fs, FALSE)); SVN_ERR(initialize_fs_struct(dst_fs)); - SVN_ERR(svn_fs_x__hotcopy_prepare_target(src_fs, dst_fs, dst_path, - incremental, scratch_pool)); - uninitialize_fs_struct(dst_fs); - /* Now, the destination repo should open just fine. */ - SVN_ERR(x_open(dst_fs, dst_path, common_pool_lock, scratch_pool, - common_pool)); - if (cancel_func) - SVN_ERR(cancel_func(cancel_baton)); - - /* Now, we may copy data as needed ... */ - return svn_fs_x__hotcopy(src_fs, dst_fs, incremental, - notify_func, notify_baton, - cancel_func, cancel_baton, scratch_pool); + /* In INCREMENTAL mode, svn_fs_x__hotcopy() will open DST_FS. + Otherwise, it's not an FS yet --- possibly just an empty dir --- so + can't be opened. + */ + return svn_fs_x__hotcopy(src_fs, dst_fs, src_path, dst_path, + incremental, notify_func, notify_baton, + cancel_func, cancel_baton, common_pool_lock, + scratch_pool, common_pool); } @@ -664,6 +665,8 @@ svn_fs_x__init(const svn_version_t *loader_version, loader_version->major); SVN_ERR(svn_ver_check_list2(x_version(), checklist, svn_ver_equal)); + SVN_ERR(svn_fs_x__batch_fsync_init(common_pool)); + *vtable = &library_vtable; return SVN_NO_ERROR; } diff --git a/subversion/libsvn_fs_x/fs.h b/subversion/libsvn_fs_x/fs.h index afb4b2a63bf3..7c7a7715567f 100644 --- a/subversion/libsvn_fs_x/fs.h +++ b/subversion/libsvn_fs_x/fs.h @@ -1,4 +1,4 @@ -/* fs.h : interface to Subversion filesystem, private to libsvn_fs +/* fs.h : interface to Subversion filesystem * * ==================================================================== * Licensed to the Apache Software Foundation (ASF) under one @@ -20,8 +20,8 @@ * ==================================================================== */ -#ifndef SVN_LIBSVN_FS_X_H -#define SVN_LIBSVN_FS_X_H +#ifndef SVN_LIBSVN_FS_X_FS_H +#define SVN_LIBSVN_FS_X_FS_H #include <apr_pools.h> #include <apr_hash.h> @@ -37,7 +37,7 @@ #include "private/svn_sqlite.h" #include "private/svn_mutex.h" -#include "id.h" +#include "rev_file.h" #ifdef __cplusplus extern "C" { @@ -53,12 +53,11 @@ extern "C" { #define PATH_FORMAT "format" /* Contains format number */ #define PATH_UUID "uuid" /* Contains UUID */ #define PATH_CURRENT "current" /* Youngest revision */ +#define PATH_NEXT "next" /* Revision begin written. */ #define PATH_LOCK_FILE "write-lock" /* Revision lock file */ #define PATH_PACK_LOCK_FILE "pack-lock" /* Pack lock file */ #define PATH_REVS_DIR "revs" /* Directory of revisions */ -#define PATH_REVPROPS_DIR "revprops" /* Directory of revprops */ #define PATH_TXNS_DIR "transactions" /* Directory of transactions */ -#define PATH_NODE_ORIGINS_DIR "node-origins" /* Lazy node-origin cache */ #define PATH_TXN_PROTOS_DIR "txn-protorevs" /* Directory of proto-revs */ #define PATH_TXN_CURRENT "txn-current" /* File with next txn key */ #define PATH_TXN_CURRENT_LOCK "txn-current-lock" /* Lock for txn-current */ @@ -81,8 +80,6 @@ extern "C" { /* Names of special files and file extensions for transactions */ #define PATH_CHANGES "changes" /* Records changes made so far */ #define PATH_TXN_PROPS "props" /* Transaction properties */ -#define PATH_TXN_PROPS_FINAL "props-final" /* Final transaction properties - before moving to revprops */ #define PATH_NEXT_IDS "next-ids" /* Next temporary ID assignments */ #define PATH_PREFIX_NODE "node." /* Prefix for node filename */ #define PATH_EXT_TXN ".txn" /* Extension of txn dir */ @@ -124,7 +121,11 @@ extern "C" { Note: If you bump this, please update the switch statement in svn_fs_x__create() as well. */ -#define SVN_FS_X__FORMAT_NUMBER 1 +#define SVN_FS_X__FORMAT_NUMBER 2 + +/* Latest experimental format number. Experimental formats are only + compatible with themselves. */ +#define SVN_FS_X__EXPERIMENTAL_FORMAT_NUMBER 2 /* On most operating systems apr implements file locks per process, not per file. On Windows apr implements the locking as per file handle @@ -136,6 +137,11 @@ extern "C" { #define SVN_FS_X__USE_LOCK_MUTEX 0 #endif +/* Maximum number of changes we deliver per request when listing the + changed paths for a given revision. Anything > 0 will do. + At 100..300 bytes per entry, this limits the allocation to ~30kB. */ +#define SVN_FS_X__CHANGES_BLOCK_SIZE 100 + /* Private FSX-specific data shared between all svn_txn_t objects that relate to a particular transaction in a filesystem (as identified by transaction id and filesystem UUID). Objects of this type are @@ -179,11 +185,15 @@ typedef struct svn_fs_x__shared_data_t declaration here. Any subset may be acquired and held at any given time but their relative acquisition order must not change. - (lock 'txn-current' before 'pack' before 'write' before 'txn-list') */ + (lock 'pack' before 'write' before 'txn-current' before 'txn-list') */ /* A lock for intra-process synchronization when accessing the TXNS list. */ svn_mutex__t *txn_list_lock; + /* A lock for intra-process synchronization when locking the + txn-current file. */ + svn_mutex__t *txn_current_lock; + /* A lock for intra-process synchronization when grabbing the repository write lock. */ svn_mutex__t *fs_write_lock; @@ -192,10 +202,6 @@ typedef struct svn_fs_x__shared_data_t repository pack operation lock. */ svn_mutex__t *fs_pack_lock; - /* A lock for intra-process synchronization when locking the - txn-current file. */ - svn_mutex__t *txn_current_lock; - /* The common pool, under which this object is allocated, subpools of which are used to allocate the transaction objects. */ apr_pool_t *common_pool; @@ -281,13 +287,9 @@ typedef struct svn_fs_x__data_t e.g. memcached may be ignored as caching is an optional feature. */ svn_boolean_t fail_stop; - /* Caches native dag_node_t* instances and acts as a 1st level cache */ + /* Caches native dag_node_t* instances */ svn_fs_x__dag_cache_t *dag_node_cache; - /* DAG node cache for immutable nodes. Maps (revision, fspath) - to (dag_node_t *). This is the 2nd level cache for DAG nodes. */ - svn_cache__t *rev_node_cache; - /* A cache of the contents of immutable directories; maps from unparsed FS ID to a apr_hash_t * mapping (const char *) dirent names to (svn_fs_x__dirent_t *). */ @@ -297,9 +299,8 @@ typedef struct svn_fs_x__data_t rep key (revision/offset) to svn_stringbuf_t. */ svn_cache__t *fulltext_cache; - /* Access object to the revprop "generation". Will be NULL until - the first access. May be also get closed and set to NULL again. */ - apr_file_t *revprop_generation_file; + /* Revprop generation number. Will be -1 if it has to reread from disk. */ + apr_int64_t revprop_generation; /* Revision property cache. Maps from (rev,generation) to apr_hash_t. */ svn_cache__t *revprop_cache; @@ -307,12 +308,6 @@ typedef struct svn_fs_x__data_t /* Node properties cache. Maps from rep key to apr_hash_t. */ svn_cache__t *properties_cache; - /* Pack manifest cache; a cache mapping (svn_revnum_t) shard number to - a manifest; and a manifest is a mapping from (svn_revnum_t) revision - number offset within a shard to (apr_off_t) byte-offset in the - respective pack file. */ - svn_cache__t *packed_offset_cache; - /* Cache for txdelta_window_t objects; * the key is svn_fs_x__window_cache_key_t */ svn_cache__t *txdelta_window_cache; @@ -329,8 +324,8 @@ typedef struct svn_fs_x__data_t the key is a (pack file revision, file offset) pair */ svn_cache__t *noderevs_container_cache; - /* Cache for change lists as APR arrays of svn_fs_x__change_t * objects; - the key is the revision */ + /* Cache for change lists n blocks as svn_fs_x__changes_list_t * objects; + the key is the (revision, first-element-in-block) pair. */ svn_cache__t *changes_cache; /* Cache for change_list_t containers; @@ -345,15 +340,6 @@ typedef struct svn_fs_x__data_t (revision, item index) pair */ svn_cache__t *rep_header_cache; - /* Cache for svn_mergeinfo_t objects; the key is a combination of - revision, inheritance flags and path. */ - svn_cache__t *mergeinfo_cache; - - /* Cache for presence of svn_mergeinfo_t on a noderev; the key is a - combination of revision, inheritance flags and path; value is "1" - if the node has mergeinfo, "0" if it doesn't. */ - svn_cache__t *mergeinfo_existence_cache; - /* Cache for l2p_header_t objects; the key is (revision, is-packed). Will be NULL for pre-format7 repos */ svn_cache__t *l2p_header_cache; @@ -417,19 +403,19 @@ typedef struct svn_fs_x__data_t or dump / load cycles). */ const char *instance_id; + /* Ensure that all filesystem changes are written to disk. */ + svn_boolean_t flush_to_disk; + /* Pointer to svn_fs_open. */ svn_error_t *(*svn_fs_open_)(svn_fs_t **, const char *, apr_hash_t *, apr_pool_t *, apr_pool_t *); + } svn_fs_x__data_t; /*** Filesystem Transaction ***/ typedef struct svn_fs_x__transaction_t { - /* property list (const char * name, svn_string_t * value). - may be NULL if there are no properties. */ - apr_hash_t *proplist; - /* revision upon which this txn is base. (unfinished only) */ svn_revnum_t base_rev; @@ -503,7 +489,8 @@ typedef struct svn_fs_x__noderev_t /* node kind */ svn_node_kind_t kind; - /* number of predecessors this node revision has (recursively). */ + /* Number of predecessors this node revision has (recursively). + A difference from the BDB backend is that it cannot be -1. */ int predecessor_count; /* representation key for this node's properties. may be NULL if @@ -543,32 +530,46 @@ typedef struct svn_fs_x__dirent_t /*** Change ***/ -typedef struct svn_fs_x__change_t +typedef svn_fs_path_change3_t svn_fs_x__change_t; + +/*** Context for reading changed paths lists iteratively. */ +typedef struct svn_fs_x__changes_context_t { - /* Path of the change. */ - svn_string_t path; + /* Repository to fetch from. */ + svn_fs_t *fs; - /* node revision id of changed path */ - svn_fs_x__id_t noderev_id; + /* Revision that we read from. */ + svn_revnum_t revision; - /* See svn_fs_path_change2_t for a description for the remaining elements. - */ - svn_fs_path_change_kind_t change_kind; + /* Revision file object to use when needed. */ + svn_fs_x__revision_file_t *revision_file; - svn_boolean_t text_mod; - svn_boolean_t prop_mod; - svn_node_kind_t node_kind; + /* Index of the next change to fetch. */ + int next; - svn_boolean_t copyfrom_known; - svn_revnum_t copyfrom_rev; - const char *copyfrom_path; + /* Offset, within the changed paths list on disk, of the next change to + fetch. */ + apr_off_t next_offset; + + /* Has the end of the list been reached? */ + svn_boolean_t eol; + +} svn_fs_x__changes_context_t; + +/*** Directory (only used at the cache interface) ***/ +typedef struct svn_fs_x__dir_data_t +{ + /* Contents, i.e. all directory entries, sorted by name. */ + apr_array_header_t *entries; - svn_tristate_t mergeinfo_mod; -} svn_fs_x__change_t; + /* SVN_INVALID_FILESIZE for committed data, otherwise the length of the + * in-txn on-disk representation of that directory. */ + svn_filesize_t txn_filesize; +} svn_fs_x__dir_data_t; #ifdef __cplusplus } #endif /* __cplusplus */ -#endif /* SVN_LIBSVN_FS_X_H */ +#endif /* SVN_LIBSVN_FS_X_FS_H */ diff --git a/subversion/libsvn_fs_x/fs_init.h b/subversion/libsvn_fs_x/fs_init.h new file mode 100644 index 000000000000..8ecbda34b078 --- /dev/null +++ b/subversion/libsvn_fs_x/fs_init.h @@ -0,0 +1,33 @@ +/* +* libsvn_fs_x/fs_init.h: Exported function of libsvn_fs_x +* +* ==================================================================== +* Licensed to the Apache Software Foundation (ASF) under one +* or more contributor license agreements. See the NOTICE file +* distributed with this work for additional information +* regarding copyright ownership. The ASF licenses this file +* to you under the Apache License, Version 2.0 (the +* "License"); you may not use this file except in compliance +* with the License. You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an +* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +* KIND, either express or implied. See the License for the +* specific language governing permissions and limitations +* under the License. +* ==================================================================== +*/ + +#ifndef LIBSVN_FS_LOADER_H +#error Please include libsvn_fs/fs_loader.h instead of this file +#else + +svn_error_t *svn_fs_x__init(const svn_version_t *loader_version, + fs_library_vtable_t **vtable, + apr_pool_t* common_pool); + +#endif + diff --git a/subversion/libsvn_fs_x/fs_x.c b/subversion/libsvn_fs_x/fs_x.c index b766b58b2201..ba1982f278bb 100644 --- a/subversion/libsvn_fs_x/fs_x.c +++ b/subversion/libsvn_fs_x/fs_x.c @@ -33,6 +33,7 @@ #include "cached_data.h" #include "id.h" +#include "low_level.h" #include "rep-cache.h" #include "revprops.h" #include "transaction.h" @@ -95,13 +96,27 @@ check_format(int format) { /* Put blacklisted versions here. */ - /* We support all formats from 1-current simultaneously */ - if (1 <= format && format <= SVN_FS_X__FORMAT_NUMBER) + /* We support any format if it matches the current format. */ + if (format == SVN_FS_X__FORMAT_NUMBER) + return SVN_NO_ERROR; + + /* Experimental formats are only supported if they match the current, but + * that case has already been handled. So, reject any experimental format. + */ + if (SVN_FS_X__EXPERIMENTAL_FORMAT_NUMBER >= format) + return svn_error_createf(SVN_ERR_FS_UNSUPPORTED_FORMAT, NULL, + _("Unsupported experimental FSX format '%d' found; current format is '%d'"), + format, SVN_FS_X__FORMAT_NUMBER); + + /* By default, we will support any non-experimental format released so far. + */ + if (format <= SVN_FS_X__FORMAT_NUMBER) return SVN_NO_ERROR; return svn_error_createf(SVN_ERR_FS_UNSUPPORTED_FORMAT, NULL, - _("Expected FS format between '1' and '%d'; found format '%d'"), - SVN_FS_X__FORMAT_NUMBER, format); + _("Expected FSX format between '%d' and '%d'; found format '%d'"), + SVN_FS_X__EXPERIMENTAL_FORMAT_NUMBER + 1, SVN_FS_X__FORMAT_NUMBER, + format); } /* Read the format file at PATH and set *PFORMAT to the format version found @@ -184,8 +199,9 @@ svn_fs_x__write_format(svn_fs_t *fs, } else { - SVN_ERR(svn_io_write_atomic(path, sb->data, sb->len, - NULL /* copy_perms_path */, scratch_pool)); + SVN_ERR(svn_io_write_atomic2(path, sb->data, sb->len, + NULL /* copy_perms_path */, + ffd->flush_to_disk, scratch_pool)); } /* And set the perms to make it read only */ @@ -519,6 +535,20 @@ write_config(svn_fs_t *fs, fsx_conf_contents, scratch_pool); } +/* Read / Evaluate the global configuration in FS->CONFIG to set up + * parameters in FS. */ +static svn_error_t * +read_global_config(svn_fs_t *fs) +{ + svn_fs_x__data_t *ffd = fs->fsap_data; + + ffd->flush_to_disk = !svn_hash__get_bool(fs->config, + SVN_FS_CONFIG_NO_FLUSH_TO_DISK, + FALSE); + + return SVN_NO_ERROR; +} + /* Read FS's UUID file and store the data in the FS struct. */ static svn_error_t * read_uuid(svn_fs_t *fs, @@ -587,8 +617,12 @@ svn_fs_x__open(svn_fs_t *fs, /* Read the configuration file. */ SVN_ERR(read_config(ffd, fs->path, fs->pool, scratch_pool)); - return svn_error_trace(svn_fs_x__read_current(&ffd->youngest_rev_cache, - fs, scratch_pool)); + /* Global configuration options. */ + SVN_ERR(read_global_config(fs)); + + ffd->youngest_rev_cache = 0; + + return SVN_NO_ERROR; } /* Baton type bridging svn_fs_x__upgrade and upgrade_body carrying @@ -840,16 +874,14 @@ static svn_error_t * write_revision_zero(svn_fs_t *fs, apr_pool_t *scratch_pool) { - /* Use an explicit sub-pool to have full control over temp file lifetimes. - * Since we have it, use it for everything else as well. */ - apr_pool_t *subpool = svn_pool_create(scratch_pool); - const char *path_revision_zero = svn_fs_x__path_rev(fs, 0, subpool); + const char *path_revision_zero = svn_fs_x__path_rev(fs, 0, scratch_pool); apr_hash_t *proplist; svn_string_t date; apr_array_header_t *index_entries; svn_fs_x__p2l_entry_t *entry; svn_fs_x__revision_file_t *rev_file; + apr_file_t *apr_file; const char *l2p_proto_index, *p2l_proto_index; /* Construct a skeleton r0 with no indexes. */ @@ -860,62 +892,72 @@ write_revision_zero(svn_fs_t *fs, "count: 0\n" "cpath: /\n" "\n", - subpool); + scratch_pool); svn_string_t *changes_str = svn_string_create("\n", - subpool); - svn_string_t *r0 = svn_string_createf(subpool, "%s%s", + scratch_pool); + svn_string_t *r0 = svn_string_createf(scratch_pool, "%s%s", noderev_str->data, changes_str->data); /* Write skeleton r0 to disk. */ - SVN_ERR(svn_io_file_create(path_revision_zero, r0->data, subpool)); + SVN_ERR(svn_io_file_create(path_revision_zero, r0->data, scratch_pool)); /* Construct the index P2L contents: describe the 2 items we have. Be sure to create them in on-disk order. */ - index_entries = apr_array_make(subpool, 2, sizeof(entry)); + index_entries = apr_array_make(scratch_pool, 2, sizeof(entry)); - entry = apr_pcalloc(subpool, sizeof(*entry)); + entry = apr_pcalloc(scratch_pool, sizeof(*entry)); entry->offset = 0; entry->size = (apr_off_t)noderev_str->len; entry->type = SVN_FS_X__ITEM_TYPE_NODEREV; entry->item_count = 1; - entry->items = apr_pcalloc(subpool, sizeof(*entry->items)); + entry->items = apr_pcalloc(scratch_pool, sizeof(*entry->items)); entry->items[0].change_set = 0; entry->items[0].number = SVN_FS_X__ITEM_INDEX_ROOT_NODE; APR_ARRAY_PUSH(index_entries, svn_fs_x__p2l_entry_t *) = entry; - entry = apr_pcalloc(subpool, sizeof(*entry)); + entry = apr_pcalloc(scratch_pool, sizeof(*entry)); entry->offset = (apr_off_t)noderev_str->len; entry->size = (apr_off_t)changes_str->len; entry->type = SVN_FS_X__ITEM_TYPE_CHANGES; entry->item_count = 1; - entry->items = apr_pcalloc(subpool, sizeof(*entry->items)); + entry->items = apr_pcalloc(scratch_pool, sizeof(*entry->items)); entry->items[0].change_set = 0; entry->items[0].number = SVN_FS_X__ITEM_INDEX_CHANGES; APR_ARRAY_PUSH(index_entries, svn_fs_x__p2l_entry_t *) = entry; /* Now re-open r0, create proto-index files from our entries and - rewrite the index section of r0. */ - SVN_ERR(svn_fs_x__open_pack_or_rev_file_writable(&rev_file, fs, 0, - subpool, subpool)); + rewrite the index section of r0. */ + SVN_ERR(svn_fs_x__rev_file_open_writable(&rev_file, fs, 0, + scratch_pool, scratch_pool)); SVN_ERR(svn_fs_x__p2l_index_from_p2l_entries(&p2l_proto_index, fs, rev_file, index_entries, - subpool, subpool)); + scratch_pool, scratch_pool)); SVN_ERR(svn_fs_x__l2p_index_from_p2l_entries(&l2p_proto_index, fs, index_entries, - subpool, subpool)); - SVN_ERR(svn_fs_x__add_index_data(fs, rev_file->file, l2p_proto_index, - p2l_proto_index, 0, subpool)); + scratch_pool, scratch_pool)); + SVN_ERR(svn_fs_x__rev_file_get(&apr_file, rev_file)); + SVN_ERR(svn_fs_x__add_index_data(fs, apr_file, l2p_proto_index, + p2l_proto_index, 0, scratch_pool)); SVN_ERR(svn_fs_x__close_revision_file(rev_file)); - SVN_ERR(svn_io_set_file_read_only(path_revision_zero, FALSE, fs->pool)); + SVN_ERR(svn_io_set_file_read_only(path_revision_zero, FALSE, scratch_pool)); /* Set a date on revision 0. */ - date.data = svn_time_to_cstring(apr_time_now(), fs->pool); + date.data = svn_time_to_cstring(apr_time_now(), scratch_pool); date.len = strlen(date.data); - proplist = apr_hash_make(fs->pool); + proplist = apr_hash_make(scratch_pool); svn_hash_sets(proplist, SVN_PROP_REVISION_DATE, &date); - return svn_fs_x__set_revision_proplist(fs, 0, proplist, fs->pool); + + SVN_ERR(svn_io_file_open(&apr_file, + svn_fs_x__path_revprops(fs, 0, scratch_pool), + APR_WRITE | APR_CREATE, APR_OS_DEFAULT, + scratch_pool)); + SVN_ERR(svn_fs_x__write_non_packed_revprops(apr_file, proplist, + scratch_pool)); + SVN_ERR(svn_io_file_close(apr_file, scratch_pool)); + + return SVN_NO_ERROR; } svn_error_t * @@ -935,14 +977,9 @@ svn_fs_x__create_file_tree(svn_fs_t *fs, /* Create the revision data directories. */ SVN_ERR(svn_io_make_dir_recursively( - svn_fs_x__path_rev_shard(fs, 0, scratch_pool), + svn_fs_x__path_shard(fs, 0, scratch_pool), scratch_pool)); - /* Create the revprops directory. */ - SVN_ERR(svn_io_make_dir_recursively( - svn_fs_x__path_revprops_shard(fs, 0, scratch_pool), - scratch_pool)); - /* Create the transaction directory. */ SVN_ERR(svn_io_make_dir_recursively( svn_fs_x__path_txns_dir(fs, scratch_pool), @@ -954,19 +991,21 @@ svn_fs_x__create_file_tree(svn_fs_t *fs, scratch_pool)); /* Create the 'current' file. */ - SVN_ERR(svn_io_file_create_empty(svn_fs_x__path_current(fs, scratch_pool), - scratch_pool)); - SVN_ERR(svn_fs_x__write_current(fs, 0, scratch_pool)); + SVN_ERR(svn_io_file_create(svn_fs_x__path_current(fs, scratch_pool), + "0\n", scratch_pool)); /* Create the 'uuid' file. */ SVN_ERR(svn_io_file_create_empty(svn_fs_x__path_lock(fs, scratch_pool), scratch_pool)); - SVN_ERR(svn_fs_x__set_uuid(fs, NULL, NULL, scratch_pool)); + SVN_ERR(svn_fs_x__set_uuid(fs, NULL, NULL, FALSE, scratch_pool)); /* Create the fsfs.conf file. */ SVN_ERR(write_config(fs, scratch_pool)); SVN_ERR(read_config(ffd, fs->path, fs->pool, scratch_pool)); + /* Global configuration options. */ + SVN_ERR(read_global_config(fs)); + /* Add revision 0. */ SVN_ERR(write_revision_zero(fs, scratch_pool)); @@ -984,6 +1023,9 @@ svn_fs_x__create_file_tree(svn_fs_t *fs, scratch_pool)); /* Initialize the revprop caching info. */ + SVN_ERR(svn_io_file_create_empty( + svn_fs_x__path_revprop_generation(fs, scratch_pool), + scratch_pool)); SVN_ERR(svn_fs_x__reset_revprop_generation_file(fs, scratch_pool)); ffd->youngest_rev_cache = 0; @@ -1040,6 +1082,7 @@ svn_error_t * svn_fs_x__set_uuid(svn_fs_t *fs, const char *uuid, const char *instance_id, + svn_boolean_t overwrite, apr_pool_t *scratch_pool) { svn_fs_x__data_t *ffd = fs->fsap_data; @@ -1058,11 +1101,23 @@ svn_fs_x__set_uuid(svn_fs_t *fs, svn_stringbuf_appendcstr(contents, "\n"); /* We use the permissions of the 'current' file, because the 'uuid' - file does not exist during repository creation. */ - SVN_ERR(svn_io_write_atomic(uuid_path, contents->data, contents->len, - /* perms */ - svn_fs_x__path_current(fs, scratch_pool), - scratch_pool)); + file does not exist during repository creation. + + svn_io_write_atomic2() does a load of magic to allow it to + replace version files that already exist. We only need to do + that when we're allowed to overwrite an existing file. */ + if (! overwrite) + { + /* Create the file */ + SVN_ERR(svn_io_file_create(uuid_path, contents->data, scratch_pool)); + } + else + { + SVN_ERR(svn_io_write_atomic2(uuid_path, contents->data, contents->len, + /* perms */ + svn_fs_x__path_current(fs, scratch_pool), + ffd->flush_to_disk, scratch_pool)); + } fs->uuid = apr_pstrdup(fs->pool, uuid); ffd->instance_id = apr_pstrdup(fs->pool, instance_id); @@ -1100,13 +1155,14 @@ svn_fs_x__revision_prop(svn_string_t **value_p, svn_fs_t *fs, svn_revnum_t rev, const char *propname, + svn_boolean_t refresh, apr_pool_t *result_pool, apr_pool_t *scratch_pool) { apr_hash_t *table; SVN_ERR(svn_fs__check_fs(fs, TRUE)); - SVN_ERR(svn_fs_x__get_revision_proplist(&table, fs, rev, FALSE, + SVN_ERR(svn_fs_x__get_revision_proplist(&table, fs, rev, FALSE, refresh, scratch_pool, scratch_pool)); *value_p = svn_string_dup(svn_hash_gets(table, propname), result_pool); @@ -1133,17 +1189,18 @@ change_rev_prop_body(void *baton, { change_rev_prop_baton_t *cb = baton; apr_hash_t *table; + const svn_string_t *present_value; /* Read current revprop values from disk (never from cache). Even if somehow the cache got out of sync, we want to make sure that we read, update and write up-to-date data. */ SVN_ERR(svn_fs_x__get_revision_proplist(&table, cb->fs, cb->rev, TRUE, - scratch_pool, scratch_pool)); + TRUE, scratch_pool, scratch_pool)); + present_value = svn_hash_gets(table, cb->name); if (cb->old_value_p) { const svn_string_t *wanted_value = *cb->old_value_p; - const svn_string_t *present_value = svn_hash_gets(table, cb->name); if ((!wanted_value != !present_value) || (wanted_value && present_value && !svn_string_compare(wanted_value, present_value))) @@ -1156,6 +1213,13 @@ change_rev_prop_body(void *baton, } /* Fall through. */ } + + /* If the prop-set is a no-op, skip the actual write. */ + if ((!present_value && !cb->value) + || (present_value && cb->value + && svn_string_compare(present_value, cb->value))) + return SVN_NO_ERROR; + svn_hash_sets(table, cb->name, cb->value); return svn_fs_x__set_revision_proplist(cb->fs, cb->rev, table, @@ -1205,8 +1269,11 @@ svn_fs_x__info_format(int *fs_format, { case 1: break; + case 2: + (*supports_version)->minor = 10; + break; #ifdef SVN_DEBUG -# if SVN_FS_X__FORMAT_NUMBER != 1 +# if SVN_FS_X__FORMAT_NUMBER != 2 # error "Need to add a 'case' statement here" # endif #endif diff --git a/subversion/libsvn_fs_x/fs_x.h b/subversion/libsvn_fs_x/fs_x.h index 98be702598c2..08d82bd81004 100644 --- a/subversion/libsvn_fs_x/fs_x.h +++ b/subversion/libsvn_fs_x/fs_x.h @@ -1,4 +1,4 @@ -/* fs_x.h : interface to the native filesystem layer +/* fs_x.h : interface to the FSX layer * * ==================================================================== * Licensed to the Apache Software Foundation (ASF) under one @@ -20,8 +20,8 @@ * ==================================================================== */ -#ifndef SVN_LIBSVN_FS__FS_X_H -#define SVN_LIBSVN_FS__FS_X_H +#ifndef SVN_LIBSVN_FS_X_FS_X_H +#define SVN_LIBSVN_FS_X_FS_X_H #include "fs.h" @@ -41,6 +41,16 @@ svn_fs_x__open(svn_fs_t *fs, const char *path, apr_pool_t *scratch_pool); +/* Initialize parts of the FS data that are being shared across multiple + filesystem objects. Use COMMON_POOL for process-wide and SCRATCH_POOL + for temporary allocations. Use COMMON_POOL_LOCK to ensure that the + initialization is serialized. */ +svn_error_t * +svn_fs_x__initialize_shared_data(svn_fs_t *fs, + svn_mutex__t *common_pool_lock, + apr_pool_t *scratch_pool, + apr_pool_t *common_pool); + /* Upgrade the fsx filesystem FS. Indicate progress via the optional * NOTIFY_FUNC callback using NOTIFY_BATON. The optional CANCEL_FUNC * will periodically be called with CANCEL_BATON to allow for preemption. @@ -138,11 +148,16 @@ svn_fs_x__create(svn_fs_t *fs, /* Set the uuid of repository FS to UUID and the instance ID to INSTANCE_ID. If any of them is NULL, use a newly generated UUID / ID instead. + + If OVERWRITE is not set, the uuid file must not exist yet implying this + is a fresh repository. + Perform temporary allocations in SCRATCH_POOL. */ svn_error_t * svn_fs_x__set_uuid(svn_fs_t *fs, const char *uuid, const char *instance_id, + svn_boolean_t overwrite, apr_pool_t *scratch_pool); /* Read the format number and maximum number of files per directory @@ -160,12 +175,15 @@ svn_fs_x__write_format(svn_fs_t *fs, /* Find the value of the property named PROPNAME in transaction REV. Return the contents in *VALUE_P, allocated from RESULT_POOL. + If REFRESH is not set, continue using the potentially outdated + revprop generation value in FS->FSAP_DATA. Use SCRATCH_POOL for temporary allocations. */ svn_error_t * svn_fs_x__revision_prop(svn_string_t **value_p, svn_fs_t *fs, svn_revnum_t rev, const char *propname, + svn_boolean_t refresh, apr_pool_t *result_pool, apr_pool_t *scratch_pool); diff --git a/subversion/libsvn_fs_x/hotcopy.c b/subversion/libsvn_fs_x/hotcopy.c index c9f0af2f7d5c..431ecb298768 100644 --- a/subversion/libsvn_fs_x/hotcopy.c +++ b/subversion/libsvn_fs_x/hotcopy.c @@ -243,6 +243,7 @@ hotcopy_io_copy_dir_recursively(svn_boolean_t *skipped_p, * to DST_SUBDIR. Assume a sharding layout based on MAX_FILES_PER_DIR. * Set *SKIPPED_P to FALSE only if the file was copied, do not change the * value in *SKIPPED_P otherwise. SKIPPED_P may be NULL if not required. + * If PROPS is set, copy the revprops file, otherwise copy the rev data file. * Use SCRATCH_POOL for temporary allocations. */ static svn_error_t * hotcopy_copy_shard_file(svn_boolean_t *skipped_p, @@ -250,6 +251,7 @@ hotcopy_copy_shard_file(svn_boolean_t *skipped_p, const char *dst_subdir, svn_revnum_t rev, int max_files_per_dir, + svn_boolean_t props, apr_pool_t *scratch_pool) { const char *src_subdir_shard = src_subdir, @@ -269,7 +271,9 @@ hotcopy_copy_shard_file(svn_boolean_t *skipped_p, SVN_ERR(hotcopy_io_dir_file_copy(skipped_p, src_subdir_shard, dst_subdir_shard, - apr_psprintf(scratch_pool, "%ld", rev), + apr_psprintf(scratch_pool, "%c%ld", + props ? 'p' : 'r', + rev), scratch_pool)); return SVN_NO_ERROR; } @@ -296,9 +300,6 @@ hotcopy_copy_packed_shard(svn_boolean_t *skipped_p, const char *dst_subdir; const char *packed_shard; const char *src_subdir_packed_shard; - svn_revnum_t revprop_rev; - apr_pool_t *iterpool; - svn_fs_x__data_t *src_ffd = src_fs->fsap_data; /* Copy the packed shard. */ src_subdir = svn_dirent_join(src_fs->path, PATH_REVS_DIR, scratch_pool); @@ -313,47 +314,6 @@ hotcopy_copy_packed_shard(svn_boolean_t *skipped_p, NULL /* cancel_func */, NULL, scratch_pool)); - /* Copy revprops belonging to revisions in this pack. */ - src_subdir = svn_dirent_join(src_fs->path, PATH_REVPROPS_DIR, scratch_pool); - dst_subdir = svn_dirent_join(dst_fs->path, PATH_REVPROPS_DIR, scratch_pool); - - if (src_ffd->min_unpacked_rev < rev + max_files_per_dir) - { - /* copy unpacked revprops rev by rev */ - iterpool = svn_pool_create(scratch_pool); - for (revprop_rev = rev; - revprop_rev < rev + max_files_per_dir; - revprop_rev++) - { - svn_pool_clear(iterpool); - - SVN_ERR(hotcopy_copy_shard_file(skipped_p, src_subdir, dst_subdir, - revprop_rev, max_files_per_dir, - iterpool)); - } - svn_pool_destroy(iterpool); - } - else - { - /* revprop for revision 0 will never be packed */ - if (rev == 0) - SVN_ERR(hotcopy_copy_shard_file(skipped_p, src_subdir, dst_subdir, - 0, max_files_per_dir, - scratch_pool)); - - /* packed revprops folder */ - packed_shard = apr_psprintf(scratch_pool, "%ld" PATH_EXT_PACKED_SHARD, - rev / max_files_per_dir); - src_subdir_packed_shard = svn_dirent_join(src_subdir, packed_shard, - scratch_pool); - SVN_ERR(hotcopy_io_copy_dir_recursively(skipped_p, - src_subdir_packed_shard, - dst_subdir, packed_shard, - TRUE /* copy_perms */, - NULL /* cancel_func */, NULL, - scratch_pool)); - } - /* If necessary, update the min-unpacked rev file in the hotcopy. */ if (*dst_min_unpacked_rev < rev + max_files_per_dir) { @@ -379,98 +339,6 @@ hotcopy_remove_file(const char *path, return SVN_NO_ERROR; } - -/* Remove revision or revprop files between START_REV (inclusive) and - * END_REV (non-inclusive) from folder DST_SUBDIR in DST_FS. Assume - * sharding as per MAX_FILES_PER_DIR. - * Use SCRATCH_POOL for temporary allocations. */ -static svn_error_t * -hotcopy_remove_files(svn_fs_t *dst_fs, - const char *dst_subdir, - svn_revnum_t start_rev, - svn_revnum_t end_rev, - int max_files_per_dir, - apr_pool_t *scratch_pool) -{ - const char *shard; - const char *dst_subdir_shard; - svn_revnum_t rev; - apr_pool_t *iterpool; - - /* Pre-compute paths for initial shard. */ - shard = apr_psprintf(scratch_pool, "%ld", start_rev / max_files_per_dir); - dst_subdir_shard = svn_dirent_join(dst_subdir, shard, scratch_pool); - - iterpool = svn_pool_create(scratch_pool); - for (rev = start_rev; rev < end_rev; rev++) - { - svn_pool_clear(iterpool); - - /* If necessary, update paths for shard. */ - if (rev != start_rev && rev % max_files_per_dir == 0) - { - shard = apr_psprintf(iterpool, "%ld", rev / max_files_per_dir); - dst_subdir_shard = svn_dirent_join(dst_subdir, shard, scratch_pool); - } - - /* remove files for REV */ - SVN_ERR(hotcopy_remove_file(svn_dirent_join(dst_subdir_shard, - apr_psprintf(iterpool, - "%ld", rev), - iterpool), - iterpool)); - } - - svn_pool_destroy(iterpool); - - return SVN_NO_ERROR; -} - -/* Remove revisions between START_REV (inclusive) and END_REV (non-inclusive) - * from DST_FS. Assume sharding as per MAX_FILES_PER_DIR. - * Use SCRATCH_POOL for temporary allocations. */ -static svn_error_t * -hotcopy_remove_rev_files(svn_fs_t *dst_fs, - svn_revnum_t start_rev, - svn_revnum_t end_rev, - int max_files_per_dir, - apr_pool_t *scratch_pool) -{ - SVN_ERR_ASSERT(start_rev <= end_rev); - SVN_ERR(hotcopy_remove_files(dst_fs, - svn_dirent_join(dst_fs->path, - PATH_REVS_DIR, - scratch_pool), - start_rev, end_rev, - max_files_per_dir, scratch_pool)); - - return SVN_NO_ERROR; -} - -/* Remove revision properties between START_REV (inclusive) and END_REV - * (non-inclusive) from DST_FS. Assume sharding as per MAX_FILES_PER_DIR. - * Use SCRATCH_POOL for temporary allocations. Revision 0 revprops will - * not be deleted. */ -static svn_error_t * -hotcopy_remove_revprop_files(svn_fs_t *dst_fs, - svn_revnum_t start_rev, - svn_revnum_t end_rev, - int max_files_per_dir, - apr_pool_t *scratch_pool) -{ - SVN_ERR_ASSERT(start_rev <= end_rev); - - /* don't delete rev 0 props */ - SVN_ERR(hotcopy_remove_files(dst_fs, - svn_dirent_join(dst_fs->path, - PATH_REVPROPS_DIR, - scratch_pool), - start_rev ? start_rev : 1, end_rev, - max_files_per_dir, scratch_pool)); - - return SVN_NO_ERROR; -} - /* Verify that DST_FS is a suitable destination for an incremental * hotcopy from SRC_FS. */ static svn_error_t * @@ -506,29 +374,6 @@ hotcopy_incremental_check_preconditions(svn_fs_t *src_fs, return SVN_NO_ERROR; } -/* Remove folder PATH. Ignore errors due to the sub-tree not being empty. - * CANCEL_FUNC and CANCEL_BATON do the usual thing. - * Use SCRATCH_POOL for temporary allocations. - */ -static svn_error_t * -remove_folder(const char *path, - svn_cancel_func_t cancel_func, - void *cancel_baton, - apr_pool_t *scratch_pool) -{ - svn_error_t *err = svn_io_remove_dir2(path, TRUE, - cancel_func, cancel_baton, - scratch_pool); - - if (err && APR_STATUS_IS_ENOTEMPTY(err->apr_err)) - { - svn_error_clear(err); - err = SVN_NO_ERROR; - } - - return svn_error_trace(err); -} - /* Copy the revision and revprop files (possibly sharded / packed) from * SRC_FS to DST_FS. Do not re-copy data which already exists in DST_FS. * When copying packed or unpacked shards, checkpoint the result in DST_FS @@ -545,8 +390,6 @@ hotcopy_revisions(svn_fs_t *src_fs, svn_boolean_t incremental, const char *src_revs_dir, const char *dst_revs_dir, - const char *src_revprops_dir, - const char *dst_revprops_dir, svn_fs_hotcopy_notify_t notify_func, void* notify_baton, svn_cancel_func_t cancel_func, @@ -624,26 +467,10 @@ hotcopy_revisions(svn_fs_t *src_fs, if (notify_func && !skipped) notify_func(notify_baton, rev, pack_end_rev, iterpool); - /* Remove revision files which are now packed. */ - if (incremental) - { - SVN_ERR(hotcopy_remove_rev_files(dst_fs, rev, - rev + max_files_per_dir, - max_files_per_dir, iterpool)); - SVN_ERR(hotcopy_remove_revprop_files(dst_fs, rev, - rev + max_files_per_dir, - max_files_per_dir, - iterpool)); - } - /* Now that all revisions have moved into the pack, the original * rev dir can be removed. */ - SVN_ERR(remove_folder(svn_fs_x__path_rev_shard(dst_fs, rev, iterpool), - cancel_func, cancel_baton, iterpool)); - if (rev > 0) - SVN_ERR(remove_folder(svn_fs_x__path_revprops_shard(dst_fs, rev, - iterpool), - cancel_func, cancel_baton, iterpool)); + SVN_ERR(svn_io_remove_dir2(svn_fs_x__path_shard(dst_fs, rev, iterpool), + TRUE, cancel_func, cancel_baton, iterpool)); } if (cancel_func) @@ -677,13 +504,12 @@ hotcopy_revisions(svn_fs_t *src_fs, /* Copy the rev file. */ SVN_ERR(hotcopy_copy_shard_file(&skipped, src_revs_dir, dst_revs_dir, - rev, max_files_per_dir, + rev, max_files_per_dir, FALSE, iterpool)); /* Copy the revprop file. */ - SVN_ERR(hotcopy_copy_shard_file(&skipped, src_revprops_dir, - dst_revprops_dir, - rev, max_files_per_dir, + SVN_ERR(hotcopy_copy_shard_file(&skipped, src_revs_dir, dst_revs_dir, + rev, max_files_per_dir, TRUE, iterpool)); /* Whenever this revision did not previously exist in the destination, @@ -729,7 +555,7 @@ typedef struct hotcopy_body_baton_t { * An incremental hotcopy copies only changed or new files to the destination, * and removes files from the destination no longer present in the source. * While the incremental hotcopy is running, readers should still be able - * to access the destintation repository without error and should not see + * to access the destination repository without error and should not see * revisions currently in progress of being copied. Readers are able to see * new fully copied revisions even if the entire incremental hotcopy procedure * has not yet completed. @@ -752,8 +578,6 @@ hotcopy_body(void *baton, void* cancel_baton = hbb->cancel_baton; svn_revnum_t src_youngest; svn_revnum_t dst_youngest; - const char *src_revprops_dir; - const char *dst_revprops_dir; const char *src_revs_dir; const char *dst_revs_dir; const char *src_subdir; @@ -793,16 +617,10 @@ hotcopy_body(void *baton, src_revs_dir = svn_dirent_join(src_fs->path, PATH_REVS_DIR, scratch_pool); dst_revs_dir = svn_dirent_join(dst_fs->path, PATH_REVS_DIR, scratch_pool); - src_revprops_dir = svn_dirent_join(src_fs->path, PATH_REVPROPS_DIR, - scratch_pool); - dst_revprops_dir = svn_dirent_join(dst_fs->path, PATH_REVPROPS_DIR, - scratch_pool); /* Ensure that the required folders exist in the destination * before actually copying the revisions and revprops. */ SVN_ERR(svn_io_make_dir_recursively(dst_revs_dir, scratch_pool)); - SVN_ERR(svn_io_make_dir_recursively(dst_revprops_dir, scratch_pool)); - if (cancel_func) SVN_ERR(cancel_func(cancel_baton)); @@ -812,7 +630,6 @@ hotcopy_body(void *baton, * revision number, but also the next-ID counters). */ SVN_ERR(hotcopy_revisions(src_fs, dst_fs, src_youngest, dst_youngest, incremental, src_revs_dir, dst_revs_dir, - src_revprops_dir, dst_revprops_dir, notify_func, notify_baton, cancel_func, cancel_baton, scratch_pool)); SVN_ERR(svn_fs_x__write_current(dst_fs, src_youngest, scratch_pool)); @@ -832,16 +649,6 @@ hotcopy_body(void *baton, cancel_func, cancel_baton, scratch_pool)); - /* Now copy the node-origins cache tree. */ - src_subdir = svn_dirent_join(src_fs->path, PATH_NODE_ORIGINS_DIR, - scratch_pool); - SVN_ERR(svn_io_check_path(src_subdir, &kind, scratch_pool)); - if (kind == svn_node_dir) - SVN_ERR(hotcopy_io_copy_dir_recursively(NULL, src_subdir, dst_fs->path, - PATH_NODE_ORIGINS_DIR, TRUE, - cancel_func, cancel_baton, - scratch_pool)); - /* * NB: Data copied below is only read by writers, not readers. * Writers are still locked out at this point. @@ -857,6 +664,10 @@ hotcopy_body(void *baton, /* Copy the rep cache and then remove entries for revisions * that did not make it into the destination. */ SVN_ERR(svn_sqlite__hotcopy(src_subdir, dst_subdir, scratch_pool)); + + /* The source might have r/o flags set on it - which would be + carried over to the copy. */ + SVN_ERR(svn_io_set_file_read_write(dst_subdir, FALSE, scratch_pool)); SVN_ERR(svn_fs_x__del_rep_reference(dst_fs, src_youngest, scratch_pool)); } @@ -871,64 +682,33 @@ hotcopy_body(void *baton, * used for the named atomics implementation. */ SVN_ERR(svn_fs_x__reset_revprop_generation_file(dst_fs, scratch_pool)); - return SVN_NO_ERROR; -} + /* Hotcopied FS is complete. Stamp it with a format file. */ + SVN_ERR(svn_fs_x__write_format(dst_fs, TRUE, scratch_pool)); -/* Wrapper around hotcopy_body taking out all necessary source repository - * locks. - */ -static svn_error_t * -hotcopy_locking_src_body(void *baton, - apr_pool_t *scratch_pool) -{ - hotcopy_body_baton_t *hbb = baton; - - return svn_error_trace(svn_fs_x__with_pack_lock(hbb->src_fs, hotcopy_body, - baton, scratch_pool)); + return SVN_NO_ERROR; } -/* Create an empty filesystem at DST_FS at DST_PATH with the same - * configuration as SRC_FS (uuid, format, and other parameters). - * After creation DST_FS has no revisions, not even revision zero. */ -static svn_error_t * -hotcopy_create_empty_dest(svn_fs_t *src_fs, - svn_fs_t *dst_fs, - const char *dst_path, - apr_pool_t *scratch_pool) +svn_error_t * +svn_fs_x__hotcopy(svn_fs_t *src_fs, + svn_fs_t *dst_fs, + const char *src_path, + const char *dst_path, + svn_boolean_t incremental, + svn_fs_hotcopy_notify_t notify_func, + void *notify_baton, + svn_cancel_func_t cancel_func, + void *cancel_baton, + svn_mutex__t *common_pool_lock, + apr_pool_t *scratch_pool, + apr_pool_t *common_pool) { - svn_fs_x__data_t *src_ffd = src_fs->fsap_data; - - /* Create the DST_FS repository with the same layout as SRC_FS. */ - SVN_ERR(svn_fs_x__create_file_tree(dst_fs, dst_path, src_ffd->format, - src_ffd->max_files_per_dir, - scratch_pool)); - - /* Copy the UUID. Hotcopy destination receives a new instance ID, but - * has the same filesystem UUID as the source. */ - SVN_ERR(svn_fs_x__set_uuid(dst_fs, src_fs->uuid, NULL, scratch_pool)); + hotcopy_body_baton_t hbb; - /* Remove revision 0 contents. Otherwise, it may not get overwritten - * due to having a newer timestamp. */ - SVN_ERR(hotcopy_remove_file(svn_fs_x__path_rev(dst_fs, 0, scratch_pool), - scratch_pool)); - SVN_ERR(hotcopy_remove_file(svn_fs_x__path_revprops(dst_fs, 0, - scratch_pool), - scratch_pool)); + if (cancel_func) + SVN_ERR(cancel_func(cancel_baton)); - /* This filesystem is ready. Stamp it with a format number. Fail if - * the 'format' file should already exist. */ - SVN_ERR(svn_fs_x__write_format(dst_fs, FALSE, scratch_pool)); + SVN_ERR(svn_fs_x__open(src_fs, src_path, scratch_pool)); - return SVN_NO_ERROR; -} - -svn_error_t * -svn_fs_x__hotcopy_prepare_target(svn_fs_t *src_fs, - svn_fs_t *dst_fs, - const char *dst_path, - svn_boolean_t incremental, - apr_pool_t *scratch_pool) -{ if (incremental) { const char *dst_format_abspath; @@ -942,40 +722,53 @@ svn_fs_x__hotcopy_prepare_target(svn_fs_t *src_fs, scratch_pool)); if (dst_format_kind == svn_node_none) { - /* Destination doesn't exist yet. Perform a normal hotcopy to a - * empty destination using the same configuration as the source. */ - SVN_ERR(hotcopy_create_empty_dest(src_fs, dst_fs, dst_path, - scratch_pool)); - } - else - { - /* Check the existing repository. */ - SVN_ERR(svn_fs_x__open(dst_fs, dst_path, scratch_pool)); - SVN_ERR(hotcopy_incremental_check_preconditions(src_fs, dst_fs)); + /* No destination? Fallback to a non-incremental hotcopy. */ + incremental = FALSE; } } + + if (incremental) + { + /* Check the existing repository. */ + SVN_ERR(svn_fs_x__open(dst_fs, dst_path, scratch_pool)); + SVN_ERR(hotcopy_incremental_check_preconditions(src_fs, dst_fs)); + + SVN_ERR(svn_fs_x__initialize_shared_data(dst_fs, common_pool_lock, + scratch_pool, common_pool)); + SVN_ERR(svn_fs_x__initialize_caches(dst_fs, scratch_pool)); + } else { /* Start out with an empty destination using the same configuration * as the source. */ - SVN_ERR(hotcopy_create_empty_dest(src_fs, dst_fs, dst_path, - scratch_pool)); + svn_fs_x__data_t *src_ffd = src_fs->fsap_data; + + /* Create the DST_FS repository with the same layout as SRC_FS. */ + SVN_ERR(svn_fs_x__create_file_tree(dst_fs, dst_path, src_ffd->format, + src_ffd->max_files_per_dir, + scratch_pool)); + + /* Copy the UUID. Hotcopy destination receives a new instance ID, but + * has the same filesystem UUID as the source. */ + SVN_ERR(svn_fs_x__set_uuid(dst_fs, src_fs->uuid, NULL, TRUE, + scratch_pool)); + + /* Remove revision 0 contents. Otherwise, it may not get overwritten + * due to having a newer timestamp. */ + SVN_ERR(hotcopy_remove_file(svn_fs_x__path_rev(dst_fs, 0, + scratch_pool), + scratch_pool)); + SVN_ERR(hotcopy_remove_file(svn_fs_x__path_revprops(dst_fs, 0, + scratch_pool), + scratch_pool)); + + SVN_ERR(svn_fs_x__initialize_shared_data(dst_fs, common_pool_lock, + scratch_pool, common_pool)); + SVN_ERR(svn_fs_x__initialize_caches(dst_fs, scratch_pool)); } - return SVN_NO_ERROR; -} - -svn_error_t * -svn_fs_x__hotcopy(svn_fs_t *src_fs, - svn_fs_t *dst_fs, - svn_boolean_t incremental, - svn_fs_hotcopy_notify_t notify_func, - void *notify_baton, - svn_cancel_func_t cancel_func, - void *cancel_baton, - apr_pool_t *scratch_pool) -{ - hotcopy_body_baton_t hbb; + if (cancel_func) + SVN_ERR(cancel_func(cancel_baton)); hbb.src_fs = src_fs; hbb.dst_fs = dst_fs; @@ -984,8 +777,16 @@ svn_fs_x__hotcopy(svn_fs_t *src_fs, hbb.notify_baton = notify_baton; hbb.cancel_func = cancel_func; hbb.cancel_baton = cancel_baton; - SVN_ERR(svn_fs_x__with_all_locks(dst_fs, hotcopy_locking_src_body, &hbb, - scratch_pool)); + + /* Lock the destination in the incremental mode. For a non-incremental + * hotcopy, don't take any locks. In that case the destination cannot be + * opened until the hotcopy finishes, and we don't have to worry about + * concurrency. */ + if (incremental) + SVN_ERR(svn_fs_x__with_all_locks(dst_fs, hotcopy_body, &hbb, + scratch_pool)); + else + SVN_ERR(hotcopy_body(&hbb, scratch_pool)); return SVN_NO_ERROR; } diff --git a/subversion/libsvn_fs_x/hotcopy.h b/subversion/libsvn_fs_x/hotcopy.h index 516c66adec8e..b4b42fa4d017 100644 --- a/subversion/libsvn_fs_x/hotcopy.h +++ b/subversion/libsvn_fs_x/hotcopy.h @@ -1,4 +1,4 @@ -/* hotcopy.h : interface to the native filesystem layer +/* hotcopy.h : interface to the hot-copying functionality * * ==================================================================== * Licensed to the Apache Software Foundation (ASF) under one @@ -20,34 +20,29 @@ * ==================================================================== */ -#ifndef SVN_LIBSVN_FS__HOTCOPY_H -#define SVN_LIBSVN_FS__HOTCOPY_H +#ifndef SVN_LIBSVN_FS_X_HOTCOPY_H +#define SVN_LIBSVN_FS_X_HOTCOPY_H #include "fs.h" -/* Create an empty copy of the fsfs filesystem SRC_FS into a new DST_FS at - * DST_PATH. If INCREMENTAL is TRUE, perform a few pre-checks only if - * a repo already exists at DST_PATH. - * Use SCRATCH_POOL for temporary allocations. */ -svn_error_t * -svn_fs_x__hotcopy_prepare_target(svn_fs_t *src_fs, - svn_fs_t *dst_fs, - const char *dst_path, - svn_boolean_t incremental, - apr_pool_t *scratch_pool); - -/* Copy the fsfs filesystem SRC_FS into DST_FS. If INCREMENTAL is TRUE, do - * not re-copy data which already exists in DST_FS. Indicate progress via - * the optional NOTIFY_FUNC callback using NOTIFY_BATON. - * Use SCRATCH_POOL for temporary allocations. */ +/* Copy the fsfs filesystem SRC_FS at SRC_PATH into a new copy DST_FS at + * DST_PATH. If INCREMENTAL is TRUE, do not re-copy data which already + * exists in DST_FS. Indicate progress via the optional NOTIFY_FUNC + * callback using NOTIFY_BATON. Use COMMON_POOL for process-wide and + * SCRATCH_POOL for temporary allocations. Use COMMON_POOL_LOCK to ensure + * that the initialization of the shared data is serialized. */ svn_error_t * svn_fs_x__hotcopy(svn_fs_t *src_fs, svn_fs_t *dst_fs, + const char *src_path, + const char *dst_path, svn_boolean_t incremental, svn_fs_hotcopy_notify_t notify_func, void *notify_baton, svn_cancel_func_t cancel_func, void *cancel_baton, - apr_pool_t *scratch_pool); + svn_mutex__t *common_pool_lock, + apr_pool_t *scratch_pool, + apr_pool_t *common_pool); #endif diff --git a/subversion/libsvn_fs_x/index.c b/subversion/libsvn_fs_x/index.c index 7d568f91872f..b7cff19f15bc 100644 --- a/subversion/libsvn_fs_x/index.c +++ b/subversion/libsvn_fs_x/index.c @@ -57,19 +57,13 @@ const apr_uint64_t off_t_max = (sizeof(apr_off_t) == sizeof(apr_int64_t)) : APR_INT32_MAX; /* We store P2L proto-index entries as 6 values, 64 bits each on disk. - * See also svn_fs_fs__p2l_proto_index_add_entry(). + * See also svn_fs_x__p2l_proto_index_add_entry(). */ #define P2L_PROTO_INDEX_ENTRY_SIZE (6 * sizeof(apr_uint64_t)) -/* We put this string in front of the L2P index header. */ -#define L2P_STREAM_PREFIX "L2P-INDEX\n" - -/* We put this string in front of the P2L index header. */ -#define P2L_STREAM_PREFIX "P2L-INDEX\n" - /* Size of the buffer that will fit the index header prefixes. */ -#define STREAM_PREFIX_LEN MAX(sizeof(L2P_STREAM_PREFIX), \ - sizeof(P2L_STREAM_PREFIX)) +#define STREAM_PREFIX_LEN MAX(sizeof(SVN_FS_X__L2P_STREAM_PREFIX), \ + sizeof(SVN_FS_X__P2L_STREAM_PREFIX)) /* Page tables in the log-to-phys index file exclusively contain entries * of this type to describe position and size of a given page. @@ -237,7 +231,7 @@ stream_error_create(svn_fs_x__packed_number_stream_t *stream, apr_off_t offset; SVN_ERR(svn_io_file_name_get(&file_name, stream->file, stream->pool)); - SVN_ERR(svn_fs_x__get_file_offset(&offset, stream->file, stream->pool)); + SVN_ERR(svn_io_file_get_offset(&offset, stream->file, stream->pool)); return svn_error_createf(err, NULL, message, file_name, apr_psprintf(stream->pool, @@ -257,7 +251,7 @@ static svn_error_t * packed_stream_read(svn_fs_x__packed_number_stream_t *stream) { unsigned char buffer[MAX_NUMBER_PREFETCH]; - apr_size_t read = 0; + apr_size_t bytes_read = 0; apr_size_t i; value_position_pair_t *target; apr_off_t block_start = 0; @@ -279,33 +273,34 @@ packed_stream_read(svn_fs_x__packed_number_stream_t *stream) * boundaries. This shall prevent jumping back and forth between two * blocks because the extra data was not actually request _now_. */ - read = sizeof(buffer); + bytes_read = sizeof(buffer); block_left = stream->block_size - (stream->next_offset - block_start); - if (block_left >= 10 && block_left < read) - read = (apr_size_t)block_left; + if (block_left >= 10 && block_left < bytes_read) + bytes_read = (apr_size_t)block_left; /* Don't read beyond the end of the file section that belongs to this * index / stream. */ - read = (apr_size_t)MIN(read, stream->stream_end - stream->next_offset); + bytes_read = (apr_size_t)MIN(bytes_read, + stream->stream_end - stream->next_offset); - err = apr_file_read(stream->file, buffer, &read); + err = apr_file_read(stream->file, buffer, &bytes_read); if (err && !APR_STATUS_IS_EOF(err)) return stream_error_create(stream, err, _("Can't read index file '%s' at offset 0x%")); /* if the last number is incomplete, trim it from the buffer */ - while (read > 0 && buffer[read-1] >= 0x80) - --read; + while (bytes_read > 0 && buffer[bytes_read-1] >= 0x80) + --bytes_read; /* we call read() only if get() requires more data. So, there must be * at least *one* further number. */ - if SVN__PREDICT_FALSE(read == 0) + if SVN__PREDICT_FALSE(bytes_read == 0) return stream_error_create(stream, err, _("Unexpected end of index file %s at offset 0x%")); /* parse file buffer and expand into stream buffer */ target = stream->buffer; - for (i = 0; i < read;) + for (i = 0; i < bytes_read;) { if (buffer[i] < 0x80) { @@ -348,20 +343,15 @@ packed_stream_read(svn_fs_x__packed_number_stream_t *stream) return SVN_NO_ERROR; } -/* Create and open a packed number stream reading from offsets START to - * END in FILE and return it in *STREAM. Access the file in chunks of - * BLOCK_SIZE bytes. Expect the stream to be prefixed by STREAM_PREFIX. - * Allocate *STREAM in RESULT_POOL and use SCRATCH_POOL for temporaries. - */ -static svn_error_t * -packed_stream_open(svn_fs_x__packed_number_stream_t **stream, - apr_file_t *file, - apr_off_t start, - apr_off_t end, - const char *stream_prefix, - apr_size_t block_size, - apr_pool_t *result_pool, - apr_pool_t *scratch_pool) +svn_error_t * +svn_fs_x__packed_stream_open(svn_fs_x__packed_number_stream_t **stream, + apr_file_t *file, + apr_off_t start, + apr_off_t end, + const char *stream_prefix, + apr_size_t block_size, + apr_pool_t *result_pool, + apr_pool_t *scratch_pool) { char buffer[STREAM_PREFIX_LEN + 1] = { 0 }; apr_size_t len = strlen(stream_prefix); @@ -516,13 +506,13 @@ read_uint64_from_proto_index(apr_file_t *proto_index, apr_pool_t *scratch_pool) { apr_byte_t buffer[sizeof(*value_p)]; - apr_size_t read; + apr_size_t bytes_read; /* Read the full 8 bytes or our 64 bit value, unless we hit EOF. * Assert that we never read partial values. */ SVN_ERR(svn_io_file_read_full2(proto_index, buffer, sizeof(buffer), - &read, eof, scratch_pool)); - SVN_ERR_ASSERT((eof && *eof) || read == sizeof(buffer)); + &bytes_read, eof, scratch_pool)); + SVN_ERR_ASSERT((eof && *eof) || bytes_read == sizeof(buffer)); /* If we did not hit EOF, reconstruct the uint64 value and return it. */ if (!eof || !*eof) @@ -702,7 +692,8 @@ svn_fs_x__l2p_proto_index_add_entry(apr_file_t *proto_index, * read operations only. */ static apr_size_t -encode_uint(unsigned char *p, apr_uint64_t value) +encode_uint(unsigned char *p, + apr_uint64_t value) { unsigned char *start = p; while (value >= 0x80) @@ -720,7 +711,8 @@ encode_uint(unsigned char *p, apr_uint64_t value) * This maps signed ints onto unsigned ones. */ static apr_size_t -encode_int(unsigned char *p, apr_int64_t value) +encode_int(unsigned char *p, + apr_int64_t value) { return encode_uint(p, (apr_uint64_t)(value < 0 ? -1 - 2*value : 2*value)); } @@ -742,7 +734,9 @@ stream_write_encoded(svn_stream_t *stream, * Return the number of remaining entries in ARRAY after START. */ static int -rle_array(apr_array_header_t *array, int start, int end) +rle_array(apr_array_header_t *array, + int start, + int end) { int i; int target = start; @@ -969,8 +963,8 @@ svn_fs_x__l2p_index_append(svn_checksum_t **checksum, /* 1 page with up to L2P_PAGE_SIZE entries. * fsfs.conf settings validation guarantees this to fit into * our address space. */ - apr_size_t last_buffer_size - = (apr_size_t)svn_spillbuf__get_size(buffer); + apr_uint64_t last_buffer_size + = (apr_uint64_t)svn_spillbuf__get_size(buffer); svn_pool_clear(iterpool); @@ -1038,7 +1032,7 @@ svn_fs_x__l2p_index_append(svn_checksum_t **checksum, /* write header info */ - SVN_ERR(svn_stream_puts(stream, L2P_STREAM_PREFIX)); + SVN_ERR(svn_stream_puts(stream, SVN_FS_X__L2P_STREAM_PREFIX)); SVN_ERR(stream_write_encoded(stream, revision)); SVN_ERR(stream_write_encoded(stream, page_counts->nelts)); SVN_ERR(stream_write_encoded(stream, ffd->l2p_page_size)); @@ -1074,7 +1068,8 @@ svn_fs_x__l2p_index_append(svn_checksum_t **checksum, * REVISION in FS. */ static svn_revnum_t -base_revision(svn_fs_t *fs, svn_revnum_t revision) +base_revision(svn_fs_t *fs, + svn_revnum_t revision) { svn_fs_x__data_t *ffd = fs->fsap_data; return svn_fs_x__is_packed_rev(fs, revision) @@ -1231,32 +1226,6 @@ expand_rle(apr_array_header_t *values, return SVN_NO_ERROR; } -/* If REV_FILE->L2P_STREAM is NULL, create a new stream for the log-to-phys - * index for REVISION in FS and return it in REV_FILE. - */ -static svn_error_t * -auto_open_l2p_index(svn_fs_x__revision_file_t *rev_file, - svn_fs_t *fs, - svn_revnum_t revision) -{ - if (rev_file->l2p_stream == NULL) - { - svn_fs_x__data_t *ffd = fs->fsap_data; - - SVN_ERR(svn_fs_x__auto_read_footer(rev_file)); - SVN_ERR(packed_stream_open(&rev_file->l2p_stream, - rev_file->file, - rev_file->l2p_offset, - rev_file->p2l_offset, - L2P_STREAM_PREFIX, - (apr_size_t)ffd->block_size, - rev_file->pool, - rev_file->pool)); - } - - return SVN_NO_ERROR; -} - /* Read the header data structure of the log-to-phys index for REVISION * in FS and return it in *HEADER, allocated in RESULT_POOL. Use REV_FILE * to access on-disk data. Use SCRATCH_POOL for temporary allocations. @@ -1279,41 +1248,48 @@ get_l2p_header_body(l2p_header_t **header, svn_revnum_t next_rev; apr_array_header_t *expanded_values = apr_array_make(scratch_pool, 16, sizeof(apr_uint64_t)); + svn_fs_x__packed_number_stream_t *stream; + svn_fs_x__rev_file_info_t file_info; + svn_fs_x__index_info_t index_info; + /* What to look for. */ svn_fs_x__pair_cache_key_t key; - key.revision = rev_file->start_revision; - key.second = rev_file->is_packed; + SVN_ERR(svn_fs_x__rev_file_info(&file_info, rev_file)); + key.revision = file_info.start_revision; + key.second = file_info.is_packed; - SVN_ERR(auto_open_l2p_index(rev_file, fs, revision)); - packed_stream_seek(rev_file->l2p_stream, 0); + /* Access the L2P index stream. */ + SVN_ERR(svn_fs_x__rev_file_l2p_index(&stream, rev_file)); + SVN_ERR(svn_fs_x__rev_file_l2p_info(&index_info, rev_file)); + packed_stream_seek(stream, 0); /* Read the table sizes. Check the data for plausibility and * consistency with other bits. */ - SVN_ERR(packed_stream_get(&value, rev_file->l2p_stream)); + SVN_ERR(packed_stream_get(&value, stream)); result->first_revision = (svn_revnum_t)value; - if (result->first_revision != rev_file->start_revision) + if (result->first_revision != file_info.start_revision) return svn_error_create(SVN_ERR_FS_INDEX_CORRUPTION, NULL, _("Index rev / pack file revision numbers do not match")); - SVN_ERR(packed_stream_get(&value, rev_file->l2p_stream)); + SVN_ERR(packed_stream_get(&value, stream)); result->revision_count = (int)value; if ( result->revision_count != 1 && result->revision_count != (apr_uint64_t)ffd->max_files_per_dir) return svn_error_create(SVN_ERR_FS_INDEX_CORRUPTION, NULL, _("Invalid number of revisions in L2P index")); - SVN_ERR(packed_stream_get(&value, rev_file->l2p_stream)); + SVN_ERR(packed_stream_get(&value, stream)); result->page_size = (apr_uint32_t)value; if (!result->page_size || (result->page_size & (result->page_size - 1))) return svn_error_create(SVN_ERR_FS_INDEX_CORRUPTION, NULL, _("L2P index page size is not a power of two")); - SVN_ERR(packed_stream_get(&value, rev_file->l2p_stream)); + SVN_ERR(packed_stream_get(&value, stream)); page_count = (apr_size_t)value; if (page_count < result->revision_count) return svn_error_create(SVN_ERR_FS_INDEX_CORRUPTION, NULL, _("Fewer L2P index pages than revisions")); - if (page_count > (rev_file->p2l_offset - rev_file->l2p_offset) / 2) + if (page_count > (index_info.end - index_info.start) / 2) return svn_error_create(SVN_ERR_FS_INDEX_CORRUPTION, NULL, _("L2P index page count implausibly large")); @@ -1333,8 +1309,7 @@ get_l2p_header_body(l2p_header_t **header, /* read per-revision page table sizes (i.e. number of pages per rev) */ page_table_index = 0; result->page_table_index[0] = page_table_index; - SVN_ERR(expand_rle(expanded_values, rev_file->l2p_stream, - result->revision_count)); + SVN_ERR(expand_rle(expanded_values, stream, result->revision_count)); for (i = 0; i < result->revision_count; ++i) { value = (apr_size_t)APR_ARRAY_IDX(expanded_values, i, apr_uint64_t); @@ -1357,13 +1332,13 @@ get_l2p_header_body(l2p_header_t **header, /* read actual page tables */ for (page = 0; page < page_count; ++page) { - SVN_ERR(packed_stream_get(&value, rev_file->l2p_stream)); + SVN_ERR(packed_stream_get(&value, stream)); if (value == 0) return svn_error_create(SVN_ERR_FS_INDEX_CORRUPTION, NULL, _("Empty L2P index page")); result->page_table[page].size = (apr_uint32_t)value; - SVN_ERR(packed_stream_get(&value, rev_file->l2p_stream)); + SVN_ERR(packed_stream_get(&value, stream)); if (value > result->page_size) return svn_error_create(SVN_ERR_FS_INDEX_CORRUPTION, NULL, _("Page exceeds L2P index page size")); @@ -1372,7 +1347,7 @@ get_l2p_header_body(l2p_header_t **header, } /* correct the page description offsets */ - offset = packed_stream_offset(rev_file->l2p_stream); + offset = packed_stream_offset(stream); for (page = 0; page < page_count; ++page) { result->page_table[page].offset = offset; @@ -1437,11 +1412,13 @@ get_l2p_header(l2p_header_t **header, { svn_fs_x__data_t *ffd = fs->fsap_data; svn_boolean_t is_cached = FALSE; + svn_fs_x__rev_file_info_t file_info; /* first, try cache lookop */ svn_fs_x__pair_cache_key_t key; - key.revision = rev_file->start_revision; - key.second = rev_file->is_packed; + SVN_ERR(svn_fs_x__rev_file_info(&file_info, rev_file)); + key.revision = file_info.start_revision; + key.second = file_info.is_packed; SVN_ERR(svn_cache__get((void**)header, &is_cached, ffd->l2p_header_cache, &key, result_pool)); if (is_cached) @@ -1454,16 +1431,12 @@ get_l2p_header(l2p_header_t **header, return SVN_NO_ERROR; } -/* From the log-to-phys index file starting at START_REVISION in FS, read - * the mapping page identified by TABLE_ENTRY and return it in *PAGE. - * Use REV_FILE to access on-disk files. - * Use RESULT_POOL for allocations. +/* From the log-to-phys index in REV_FILE, read the mapping page identified + * by TABLE_ENTRY and return it in *PAGE, allocated in RESULT_POOL. */ static svn_error_t * get_l2p_page(l2p_page_t **page, svn_fs_x__revision_file_t *rev_file, - svn_fs_t *fs, - svn_revnum_t start_revision, l2p_page_table_entry_t *table_entry, apr_pool_t *result_pool) { @@ -1472,10 +1445,11 @@ get_l2p_page(l2p_page_t **page, l2p_page_t *result = apr_pcalloc(result_pool, sizeof(*result)); apr_uint64_t container_count; apr_off_t *container_offsets; + svn_fs_x__packed_number_stream_t *stream; /* open index file and select page */ - SVN_ERR(auto_open_l2p_index(rev_file, fs, start_revision)); - packed_stream_seek(rev_file->l2p_stream, table_entry->offset); + SVN_ERR(svn_fs_x__rev_file_l2p_index(&stream, rev_file)); + packed_stream_seek(stream, table_entry->offset); /* initialize the page content */ result->entry_count = table_entry->entry_count; @@ -1486,12 +1460,12 @@ get_l2p_page(l2p_page_t **page, /* container offsets array */ - SVN_ERR(packed_stream_get(&container_count, rev_file->l2p_stream)); + SVN_ERR(packed_stream_get(&container_count, stream)); container_offsets = apr_pcalloc(result_pool, container_count * sizeof(*result)); for (i = 0; i < container_count; ++i) { - SVN_ERR(packed_stream_get(&value, rev_file->l2p_stream)); + SVN_ERR(packed_stream_get(&value, stream)); last_value += value; container_offsets[i] = (apr_off_t)last_value - 1; /* '-1' is represented as '0' in the index file */ @@ -1500,7 +1474,7 @@ get_l2p_page(l2p_page_t **page, /* read all page entries (offsets in rev file and container sub-items) */ for (i = 0; i < result->entry_count; ++i) { - SVN_ERR(packed_stream_get(&value, rev_file->l2p_stream)); + SVN_ERR(packed_stream_get(&value, stream)); if (value == 0) { result->offsets[i] = -1; @@ -1509,7 +1483,7 @@ get_l2p_page(l2p_page_t **page, else if (value <= container_count) { result->offsets[i] = container_offsets[value - 1]; - SVN_ERR(packed_stream_get(&value, rev_file->l2p_stream)); + SVN_ERR(packed_stream_get(&value, stream)); result->sub_items[i] = (apr_uint32_t)value; } else @@ -1521,7 +1495,7 @@ get_l2p_page(l2p_page_t **page, /* After reading all page entries, the read cursor must have moved by * TABLE_ENTRY->SIZE bytes. */ - if ( packed_stream_offset(rev_file->l2p_stream) + if ( packed_stream_offset(stream) != table_entry->offset + table_entry->size) return svn_error_create(SVN_ERR_FS_INDEX_CORRUPTION, NULL, _("L2P actual page size does not match page table value.")); @@ -1690,9 +1664,8 @@ get_l2p_page_table(apr_array_header_t *pages, /* Utility function. Read the l2p index pages for REVISION in FS from * STREAM and put them into the cache. Skip page number EXLCUDED_PAGE_NO * (use -1 for 'skip none') and pages outside the MIN_OFFSET, MAX_OFFSET - * range in the l2p index file. The index is being identified by - * FIRST_REVISION. PAGES is a scratch container provided by the caller. - * SCRATCH_POOL is used for temporary allocations. + * range in the l2p index file. PAGES is a scratch container provided by + * the caller. SCRATCH_POOL is used for temporary allocations. * * This function may be a no-op if the header cache lookup fails / misses. */ @@ -1700,7 +1673,6 @@ static svn_error_t * prefetch_l2p_pages(svn_boolean_t *end, svn_fs_t *fs, svn_fs_x__revision_file_t *rev_file, - svn_revnum_t first_revision, svn_revnum_t revision, apr_array_header_t *pages, int exlcuded_page_no, @@ -1769,8 +1741,7 @@ prefetch_l2p_pages(svn_boolean_t *end, /* no in cache -> read from stream (data already buffered in APR) * and cache the result */ l2p_page_t *page = NULL; - SVN_ERR(get_l2p_page(&page, rev_file, fs, first_revision, - entry, iterpool)); + SVN_ERR(get_l2p_page(&page, rev_file, entry, iterpool)); SVN_ERR(svn_cache__set(ffd->l2p_page_cache, &key, page, iterpool)); @@ -1841,8 +1812,7 @@ l2p_index_lookup(apr_off_t *offset, apr_off_t min_offset = max_offset - ffd->block_size; /* read the relevant page */ - SVN_ERR(get_l2p_page(&page, rev_file, fs, info_baton.first_revision, - &info_baton.entry, scratch_pool)); + SVN_ERR(get_l2p_page(&page, rev_file, &info_baton.entry, scratch_pool)); /* cache the page and extract the result we need */ SVN_ERR(svn_cache__set(ffd->l2p_page_cache, &key, page, scratch_pool)); @@ -1863,7 +1833,6 @@ l2p_index_lookup(apr_off_t *offset, svn_pool_clear(iterpool); SVN_ERR(prefetch_l2p_pages(&end, fs, rev_file, - info_baton.first_revision, prefetch_revision, pages, excluded_page_no, min_offset, max_offset, iterpool)); @@ -1877,7 +1846,6 @@ l2p_index_lookup(apr_off_t *offset, svn_pool_clear(iterpool); SVN_ERR(prefetch_l2p_pages(&end, fs, rev_file, - info_baton.first_revision, prefetch_revision, pages, -1, min_offset, max_offset, iterpool)); } @@ -1950,8 +1918,7 @@ svn_fs_x__l2p_get_max_ids(apr_array_header_t **max_ids, apr_pool_t *header_pool = svn_pool_create(scratch_pool); /* read index master data structure for the index covering START_REV */ - SVN_ERR(svn_fs_x__open_pack_or_rev_file(&rev_file, fs, start_rev, - header_pool, header_pool)); + SVN_ERR(svn_fs_x__rev_file_init(&rev_file, fs, start_rev, header_pool)); SVN_ERR(get_l2p_header(&header, rev_file, fs, start_rev, header_pool, header_pool)); SVN_ERR(svn_fs_x__close_revision_file(rev_file)); @@ -1965,15 +1932,15 @@ svn_fs_x__l2p_get_max_ids(apr_array_header_t **max_ids, apr_uint64_t item_count; apr_size_t first_page_index, last_page_index; - if (revision >= header->first_revision + header->revision_count) + if (revision - header->first_revision >= header->revision_count) { /* need to read the next index. Clear up memory used for the * previous one. Note that intermittent pack runs do not change * the number of items in a revision, i.e. there is no consistency * issue here. */ svn_pool_clear(header_pool); - SVN_ERR(svn_fs_x__open_pack_or_rev_file(&rev_file, fs, revision, - header_pool, header_pool)); + SVN_ERR(svn_fs_x__rev_file_init(&rev_file, fs, revision, + header_pool)); SVN_ERR(get_l2p_header(&header, rev_file, fs, revision, header_pool, header_pool)); SVN_ERR(svn_fs_x__close_revision_file(rev_file)); @@ -2215,8 +2182,8 @@ svn_fs_x__p2l_index_append(svn_checksum_t **checksum, apr_uint64_t last_entry_end = 0; apr_uint64_t last_page_end = 0; - apr_size_t last_buffer_size = 0; /* byte offset in the spill buffer at - the begin of the current revision */ + apr_uint64_t last_buffer_size = 0; /* byte offset in the spill buffer at + the begin of the current revision */ apr_uint64_t file_size = 0; /* temporary data structures that collect the data which will be moved @@ -2315,7 +2282,8 @@ svn_fs_x__p2l_index_append(svn_checksum_t **checksum, encode_uint(encoded, entry.size), iterpool)); SVN_ERR(svn_spillbuf__write(buffer, (const char *)encoded, - encode_uint(encoded, entry.type + entry.item_count * 16), + encode_uint(encoded, entry.type + + entry.item_count * 16), iterpool)); SVN_ERR(svn_spillbuf__write(buffer, (const char *)encoded, encode_uint(encoded, entry.fnv1_checksum), @@ -2359,7 +2327,7 @@ svn_fs_x__p2l_index_append(svn_checksum_t **checksum, result_pool); /* write the start revision, file size and page size */ - SVN_ERR(svn_stream_puts(stream, P2L_STREAM_PREFIX)); + SVN_ERR(svn_stream_puts(stream, SVN_FS_X__P2L_STREAM_PREFIX)); SVN_ERR(stream_write_encoded(stream, revision)); SVN_ERR(stream_write_encoded(stream, file_size)); SVN_ERR(stream_write_encoded(stream, page_size)); @@ -2382,32 +2350,6 @@ svn_fs_x__p2l_index_append(svn_checksum_t **checksum, return SVN_NO_ERROR; } -/* If REV_FILE->P2L_STREAM is NULL, create a new stream for the phys-to-log - * index for REVISION in FS using the rev / pack file provided by REV_FILE. - */ -static svn_error_t * -auto_open_p2l_index(svn_fs_x__revision_file_t *rev_file, - svn_fs_t *fs, - svn_revnum_t revision) -{ - if (rev_file->p2l_stream == NULL) - { - svn_fs_x__data_t *ffd = fs->fsap_data; - - SVN_ERR(svn_fs_x__auto_read_footer(rev_file)); - SVN_ERR(packed_stream_open(&rev_file->p2l_stream, - rev_file->file, - rev_file->p2l_offset, - rev_file->footer_offset, - P2L_STREAM_PREFIX, - (apr_size_t)ffd->block_size, - rev_file->pool, - rev_file->pool)); - } - - return SVN_NO_ERROR; -} - /* Data structure that describes which p2l page info shall be extracted * from the cache and contains the fields that receive the result. */ @@ -2515,11 +2457,15 @@ get_p2l_header(p2l_header_t **header, apr_off_t offset; p2l_header_t *result; svn_boolean_t is_cached = FALSE; + svn_fs_x__packed_number_stream_t *stream; + svn_fs_x__rev_file_info_t file_info; + svn_fs_x__index_info_t l2p_index_info; /* look for the header data in our cache */ svn_fs_x__pair_cache_key_t key; - key.revision = rev_file->start_revision; - key.second = rev_file->is_packed; + SVN_ERR(svn_fs_x__rev_file_info(&file_info, rev_file)); + key.revision = file_info.start_revision; + key.second = file_info.is_packed; SVN_ERR(svn_cache__get((void**)header, &is_cached, ffd->p2l_header_cache, &key, result_pool)); @@ -2528,32 +2474,33 @@ get_p2l_header(p2l_header_t **header, /* not found -> must read it from disk. * Open index file or position read pointer to the begin of the file */ - SVN_ERR(auto_open_p2l_index(rev_file, fs, key.revision)); - packed_stream_seek(rev_file->p2l_stream, 0); + SVN_ERR(svn_fs_x__rev_file_p2l_index(&stream, rev_file)); + SVN_ERR(svn_fs_x__rev_file_l2p_info(&l2p_index_info, rev_file)); + packed_stream_seek(stream, 0); /* allocate result data structure */ result = apr_pcalloc(result_pool, sizeof(*result)); /* Read table sizes, check them for plausibility and allocate page array. */ - SVN_ERR(packed_stream_get(&value, rev_file->p2l_stream)); + SVN_ERR(packed_stream_get(&value, stream)); result->first_revision = (svn_revnum_t)value; - if (result->first_revision != rev_file->start_revision) + if (result->first_revision != file_info.start_revision) return svn_error_create(SVN_ERR_FS_INDEX_CORRUPTION, NULL, _("Index rev / pack file revision numbers do not match")); - SVN_ERR(packed_stream_get(&value, rev_file->p2l_stream)); + SVN_ERR(packed_stream_get(&value, stream)); result->file_size = value; - if (result->file_size != (apr_uint64_t)rev_file->l2p_offset) + if (result->file_size != (apr_uint64_t)l2p_index_info.start) return svn_error_create(SVN_ERR_FS_INDEX_CORRUPTION, NULL, _("Index offset and rev / pack file size do not match")); - SVN_ERR(packed_stream_get(&value, rev_file->p2l_stream)); + SVN_ERR(packed_stream_get(&value, stream)); result->page_size = value; if (!result->page_size || (result->page_size & (result->page_size - 1))) return svn_error_create(SVN_ERR_FS_INDEX_CORRUPTION, NULL, _("P2L index page size is not a power of two")); - SVN_ERR(packed_stream_get(&value, rev_file->p2l_stream)); + SVN_ERR(packed_stream_get(&value, stream)); result->page_count = (apr_size_t)value; if (result->page_count != (result->file_size - 1) / result->page_size + 1) return svn_error_create(SVN_ERR_FS_INDEX_CORRUPTION, NULL, @@ -2566,12 +2513,12 @@ get_p2l_header(p2l_header_t **header, result->offsets[0] = 0; for (i = 0; i < result->page_count; ++i) { - SVN_ERR(packed_stream_get(&value, rev_file->p2l_stream)); + SVN_ERR(packed_stream_get(&value, stream)); result->offsets[i+1] = result->offsets[i] + (apr_off_t)value; } /* correct the offset values */ - offset = packed_stream_offset(rev_file->p2l_stream); + offset = packed_stream_offset(stream); for (i = 0; i <= result->page_count; ++i) result->offsets[i] += offset; @@ -2703,6 +2650,13 @@ read_entry(svn_fs_x__packed_number_stream_t *stream, } } + /* Corrupted SIZE values might cause arithmetic overflow. + * The same can happen if you copy a repository from a system with 63 bit + * file lengths to one with 31 bit file lengths. */ + if ((apr_uint64_t)entry.offset + (apr_uint64_t)entry.size > off_t_max) + return svn_error_create(SVN_ERR_FS_INDEX_OVERFLOW , NULL, + _("P2L index entry size overflow.")); + APR_ARRAY_PUSH(result, svn_fs_x__p2l_entry_t) = entry; *item_offset += entry.size; @@ -2732,14 +2686,15 @@ get_p2l_page(apr_array_header_t **entries, = apr_array_make(result_pool, 16, sizeof(svn_fs_x__p2l_entry_t)); apr_off_t item_offset; apr_off_t offset; + svn_fs_x__packed_number_stream_t *stream; /* open index and navigate to page start */ - SVN_ERR(auto_open_p2l_index(rev_file, fs, start_revision)); - packed_stream_seek(rev_file->p2l_stream, start_offset); + SVN_ERR(svn_fs_x__rev_file_p2l_index(&stream, rev_file)); + packed_stream_seek(stream, start_offset); /* read rev file offset of the first page entry (all page entries will * only store their sizes). */ - SVN_ERR(packed_stream_get(&value, rev_file->p2l_stream)); + SVN_ERR(packed_stream_get(&value, stream)); item_offset = (apr_off_t)value; /* Special case: empty pages. */ @@ -2747,17 +2702,15 @@ get_p2l_page(apr_array_header_t **entries, { /* Empty page. This only happens if the first entry of the next page * also covers this page (and possibly more) completely. */ - SVN_ERR(read_entry(rev_file->p2l_stream, &item_offset, start_revision, - result)); + SVN_ERR(read_entry(stream, &item_offset, start_revision, result)); } else { /* Read non-empty page. */ do { - SVN_ERR(read_entry(rev_file->p2l_stream, &item_offset, - start_revision, result)); - offset = packed_stream_offset(rev_file->p2l_stream); + SVN_ERR(read_entry(stream, &item_offset, start_revision, result)); + offset = packed_stream_offset(stream); } while (offset < next_offset); @@ -2771,9 +2724,9 @@ get_p2l_page(apr_array_header_t **entries, * entry of the next page */ if (item_offset < page_start + page_size) { - SVN_ERR(packed_stream_get(&value, rev_file->p2l_stream)); + SVN_ERR(packed_stream_get(&value, stream)); item_offset = (apr_off_t)value; - SVN_ERR(read_entry(rev_file->p2l_stream, &item_offset, + SVN_ERR(read_entry(stream, &item_offset, start_revision, result)); } } @@ -3203,7 +3156,8 @@ svn_fs_x__p2l_index_lookup(apr_array_header_t **entries, * RHS. */ static int -compare_p2l_entry_offsets(const void *lhs, const void *rhs) +compare_p2l_entry_offsets(const void *lhs, + const void *rhs) { const svn_fs_x__p2l_entry_t *entry = (const svn_fs_x__p2l_entry_t *)lhs; apr_off_t offset = *(const apr_off_t *)rhs; @@ -3212,7 +3166,7 @@ compare_p2l_entry_offsets(const void *lhs, const void *rhs) } /* Cached data extraction utility. DATA is a P2L index page, e.g. an APR - * array of svn_fs_fs__p2l_entry_t elements. Return the entry for the item, + * array of svn_fs_x__p2l_entry_t elements. Return the entry for the item, * allocated in RESULT_POOL, starting at OFFSET or NULL if that's not an * the start offset of any item. Use SCRATCH_POOL for temporary allocations. */ @@ -3503,15 +3457,13 @@ calc_fnv1(svn_fs_x__p2l_entry_t *entry, } /* Read the block and feed it to the checksum calculator. */ - SVN_ERR(svn_io_file_seek(rev_file->file, APR_SET, &entry->offset, - scratch_pool)); + SVN_ERR(svn_fs_x__rev_file_seek(rev_file, NULL, entry->offset)); while (size > 0) { apr_size_t to_read = size > sizeof(buffer) ? sizeof(buffer) : (apr_size_t)size; - SVN_ERR(svn_io_file_read_full2(rev_file->file, buffer, to_read, NULL, - NULL, scratch_pool)); + SVN_ERR(svn_fs_x__rev_file_read(rev_file, buffer, to_read)); SVN_ERR(svn_checksum_update(context, buffer, to_read)); size -= to_read; } @@ -3795,7 +3747,7 @@ svn_error_t * svn_fs_x__deserialize_l2p_header(void **out, void *data, apr_size_t data_len, - apr_pool_t *pool) + apr_pool_t *result_pool) { l2p_header_t *header = (l2p_header_t *)data; @@ -3849,7 +3801,7 @@ svn_error_t * svn_fs_x__deserialize_l2p_page(void **out, void *data, apr_size_t data_len, - apr_pool_t *pool) + apr_pool_t *result_pool) { l2p_page_t *page = data; @@ -3898,7 +3850,7 @@ svn_error_t * svn_fs_x__deserialize_p2l_header(void **out, void *data, apr_size_t data_len, - apr_pool_t *pool) + apr_pool_t *result_pool) { p2l_header_t *header = data; @@ -3956,7 +3908,7 @@ svn_error_t * svn_fs_x__deserialize_p2l_page(void **out, void *data, apr_size_t data_len, - apr_pool_t *pool) + apr_pool_t *result_pool) { apr_array_header_t *page = (apr_array_header_t *)data; svn_fs_x__p2l_entry_t *entries; @@ -3971,7 +3923,7 @@ svn_fs_x__deserialize_p2l_page(void **out, svn_temp_deserializer__resolve(entries, (void**)&entries[i].items); /* patch up members */ - page->pool = pool; + page->pool = result_pool; page->nalloc = page->nelts; /* done */ diff --git a/subversion/libsvn_fs_x/index.h b/subversion/libsvn_fs_x/index.h index 4e0e1ddfcdf2..9a5454d01118 100644 --- a/subversion/libsvn_fs_x/index.h +++ b/subversion/libsvn_fs_x/index.h @@ -20,8 +20,8 @@ * ==================================================================== */ -#ifndef SVN_LIBSVN_FS__INDEX_H -#define SVN_LIBSVN_FS__INDEX_H +#ifndef SVN_LIBSVN_FS_X_INDEX_H +#define SVN_LIBSVN_FS_X_INDEX_H #include "fs.h" #include "rev_file.h" @@ -53,6 +53,28 @@ #define SVN_FS_X__ITEM_TYPE_REPS_CONT 10 /* item is a representations container */ +/* We put this string in front of the L2P index header. */ +#define SVN_FS_X__L2P_STREAM_PREFIX "L2P-INDEX\n" + +/* We put this string in front of the P2L index header. */ +#define SVN_FS_X__P2L_STREAM_PREFIX "P2L-INDEX\n" + + +/* Create and open a packed number stream reading from offsets START to + * END in FILE and return it in *STREAM. Access the file in chunks of + * BLOCK_SIZE bytes. Expect the stream to be prefixed by STREAM_PREFIX. + * Allocate *STREAM in RESULT_POOL and use SCRATCH_POOL for temporaries. + */ +svn_error_t * +svn_fs_x__packed_stream_open(svn_fs_x__packed_number_stream_t **stream, + apr_file_t *file, + apr_off_t start, + apr_off_t end, + const char *stream_prefix, + apr_size_t block_size, + apr_pool_t *result_pool, + apr_pool_t *scratch_pool); + /* (user visible) entry in the phys-to-log index. It describes a section * of some packed / non-packed rev file as containing a specific item. * There must be no overlapping / conflicting entries. @@ -184,7 +206,7 @@ svn_fs_x__p2l_index_append(svn_checksum_t **checksum, /* Use the phys-to-log mapping files in FS to build a list of entries * that (at least partly) overlap with the range given by BLOCK_START * offset and BLOCK_SIZE in the rep / pack file containing REVISION. - * Return the array in *ENTRIES with svn_fs_fs__p2l_entry_t as elements, + * Return the array in *ENTRIES with svn_fs_x__p2l_entry_t as elements, * allocated in RESULT_POOL. REV_FILE determines whether to access single * rev or pack file data. If that is not available anymore (neither in * cache nor on disk), return an error. Use SCRATCH_POOL for temporary @@ -287,7 +309,7 @@ svn_fs_x__p2l_get_max_offset(apr_off_t *offset, /* For FS, create a new L2P auto-deleting proto index file in POOL and return * its name in *PROTONAME. All entries to write are given in ENTRIES and - * entries are of type svn_fs_fs__p2l_entry_t* (sic!). The ENTRIES array + * entries are of type svn_fs_x__p2l_entry_t* (sic!). The ENTRIES array * will be reordered. Give the proto index file the lifetime of RESULT_POOL * and use SCRATCH_POOL for temporary allocations. */ @@ -300,7 +322,7 @@ svn_fs_x__l2p_index_from_p2l_entries(const char **protoname, /* For FS, create a new P2L auto-deleting proto index file in POOL and return * its name in *PROTONAME. All entries to write are given in ENTRIES and - * of type svn_fs_fs__p2l_entry_t*. The FVN1 checksums are not taken from + * of type svn_fs_x__p2l_entry_t*. The FVN1 checksums are not taken from * ENTRIES but are begin calculated from the current contents of REV_FILE * as we go. Give the proto index file the lifetime of RESULT_POOL and use * SCRATCH_POOL for temporary allocations. @@ -350,7 +372,7 @@ svn_error_t * svn_fs_x__deserialize_l2p_header(void **out, void *data, apr_size_t data_len, - apr_pool_t *pool); + apr_pool_t *result_pool); /* * Implements svn_cache__serialize_func_t for l2p_page_t objects. @@ -368,7 +390,7 @@ svn_error_t * svn_fs_x__deserialize_l2p_page(void **out, void *data, apr_size_t data_len, - apr_pool_t *pool); + apr_pool_t *result_pool); /* * Implements svn_cache__serialize_func_t for p2l_header_t objects. @@ -386,7 +408,7 @@ svn_error_t * svn_fs_x__deserialize_p2l_header(void **out, void *data, apr_size_t data_len, - apr_pool_t *pool); + apr_pool_t *result_pool); /* * Implements svn_cache__serialize_func_t for apr_array_header_t objects @@ -406,6 +428,6 @@ svn_error_t * svn_fs_x__deserialize_p2l_page(void **out, void *data, apr_size_t data_len, - apr_pool_t *pool); + apr_pool_t *result_pool); #endif diff --git a/subversion/libsvn_fs_x/lock.c b/subversion/libsvn_fs_x/lock.c index 6819f630ef1a..d1db492c0f37 100644 --- a/subversion/libsvn_fs_x/lock.c +++ b/subversion/libsvn_fs_x/lock.c @@ -113,7 +113,8 @@ hash_fetch(apr_hash_t *hash, /* SVN_ERR_FS_CORRUPT: the lockfile for PATH in FS is corrupt. */ static svn_error_t * -err_corrupt_lockfile(const char *fs_path, const char *path) +err_corrupt_lockfile(const char *fs_path, + const char *path) { return svn_error_createf( @@ -234,7 +235,7 @@ write_digest_file(apr_hash_t *children, if ((err = svn_hash_write2(hash, stream, SVN_HASH_TERMINATOR, scratch_pool))) { - svn_error_clear(svn_stream_close(stream)); + err = svn_error_compose_create(err, svn_stream_close(stream)); return svn_error_createf(err->apr_err, err, _("Cannot write lock/entries hashfile '%s'"), @@ -243,7 +244,7 @@ write_digest_file(apr_hash_t *children, } SVN_ERR(svn_stream_close(stream)); - SVN_ERR(svn_io_file_rename(tmp_path, digest_path, scratch_pool)); + SVN_ERR(svn_io_file_rename2(tmp_path, digest_path, FALSE, scratch_pool)); SVN_ERR(svn_io_copy_perms(perms_reference, digest_path, scratch_pool)); return SVN_NO_ERROR; } @@ -286,7 +287,7 @@ read_digest_file(apr_hash_t **children_p, hash = apr_hash_make(pool); if ((err = svn_hash_read2(hash, stream, SVN_HASH_TERMINATOR, pool))) { - svn_error_clear(svn_stream_close(stream)); + err = svn_error_compose_create(err, svn_stream_close(stream)); return svn_error_createf(err->apr_err, err, _("Can't parse lock/entries hashfile '%s'"), @@ -470,6 +471,12 @@ unlock_single(svn_fs_t *fs, svn_lock_t *lock, apr_pool_t *pool); +/* Check if LOCK has been already expired. */ +static svn_boolean_t lock_expired(const svn_lock_t *lock) +{ + return lock->expiration_date && (apr_time_now() > lock->expiration_date); +} + /* Set *LOCK_P to the lock for PATH in FS. HAVE_WRITE_LOCK should be TRUE if the caller (or one of its callers) has taken out the repository-wide write lock, FALSE otherwise. If MUST_EXIST is @@ -499,7 +506,7 @@ get_lock(svn_lock_t **lock_p, return must_exist ? SVN_FS__ERR_NO_SUCH_LOCK(fs, path) : SVN_NO_ERROR; /* Don't return an expired lock. */ - if (lock->expiration_date && (apr_time_now() > lock->expiration_date)) + if (lock_expired(lock)) { /* Only remove the lock if we have the write lock. Read operations shouldn't change the filesystem. */ @@ -546,68 +553,17 @@ get_lock_helper(svn_fs_t *fs, } -/* Baton for locks_walker(). */ -typedef struct walk_locks_baton_t -{ - svn_fs_get_locks_callback_t get_locks_func; - void *get_locks_baton; - svn_fs_t *fs; -} walk_locks_baton_t; - -/* Implements walk_digests_callback_t. */ -static svn_error_t * -locks_walker(void *baton, - const char *fs_path, - const char *digest_path, - svn_lock_t *lock, - svn_boolean_t have_write_lock, - apr_pool_t *pool) -{ - walk_locks_baton_t *wlb = baton; - - if (lock) - { - /* Don't report an expired lock. */ - if (lock->expiration_date == 0 - || (apr_time_now() <= lock->expiration_date)) - { - if (wlb->get_locks_func) - SVN_ERR(wlb->get_locks_func(wlb->get_locks_baton, lock, pool)); - } - else - { - /* Only remove the lock if we have the write lock. - Read operations shouldn't change the filesystem. */ - if (have_write_lock) - SVN_ERR(unlock_single(wlb->fs, lock, pool)); - } - } - - return SVN_NO_ERROR; -} - -/* Callback type for walk_digest_files(). - * - * LOCK come from a read_digest_file(digest_path) call. - */ -typedef svn_error_t *(*walk_digests_callback_t)(void *baton, - const char *fs_path, - const char *digest_path, - svn_lock_t *lock, - svn_boolean_t have_write_lock, - apr_pool_t *pool); - -/* A function that calls WALK_DIGESTS_FUNC/WALK_DIGESTS_BATON for - all lock digest files in and under PATH in FS. +/* A function that calls GET_LOCKS_FUNC/GET_LOCKS_BATON for + all locks in and under PATH in FS. HAVE_WRITE_LOCK should be true if the caller (directly or indirectly) has the FS write lock. */ static svn_error_t * -walk_digest_files(const char *fs_path, - const char *digest_path, - walk_digests_callback_t walk_digests_func, - void *walk_digests_baton, - svn_boolean_t have_write_lock, - apr_pool_t *pool) +walk_locks(svn_fs_t *fs, + const char *digest_path, + svn_fs_get_locks_callback_t get_locks_func, + void *get_locks_baton, + svn_boolean_t have_write_lock, + apr_pool_t *pool) { apr_hash_index_t *hi; apr_hash_t *children; @@ -615,10 +571,19 @@ walk_digest_files(const char *fs_path, svn_lock_t *lock; /* First, send up any locks in the current digest file. */ - SVN_ERR(read_digest_file(&children, &lock, fs_path, digest_path, pool)); + SVN_ERR(read_digest_file(&children, &lock, fs->path, digest_path, pool)); - SVN_ERR(walk_digests_func(walk_digests_baton, fs_path, digest_path, lock, - have_write_lock, pool)); + if (lock && lock_expired(lock)) + { + /* Only remove the lock if we have the write lock. + Read operations shouldn't change the filesystem. */ + if (have_write_lock) + SVN_ERR(unlock_single(fs, lock, pool)); + } + else if (lock) + { + SVN_ERR(get_locks_func(get_locks_baton, lock, pool)); + } /* Now, report all the child entries (if any; bail otherwise). */ if (! apr_hash_count(children)) @@ -630,39 +595,25 @@ walk_digest_files(const char *fs_path, svn_pool_clear(subpool); SVN_ERR(read_digest_file - (NULL, &lock, fs_path, - digest_path_from_digest(fs_path, digest, subpool), subpool)); + (NULL, &lock, fs->path, + digest_path_from_digest(fs->path, digest, subpool), subpool)); - SVN_ERR(walk_digests_func(walk_digests_baton, fs_path, digest_path, lock, - have_write_lock, subpool)); + if (lock && lock_expired(lock)) + { + /* Only remove the lock if we have the write lock. + Read operations shouldn't change the filesystem. */ + if (have_write_lock) + SVN_ERR(unlock_single(fs, lock, pool)); + } + else if (lock) + { + SVN_ERR(get_locks_func(get_locks_baton, lock, pool)); + } } svn_pool_destroy(subpool); return SVN_NO_ERROR; } -/* A function that calls GET_LOCKS_FUNC/GET_LOCKS_BATON for - all locks in and under PATH in FS. - HAVE_WRITE_LOCK should be true if the caller (directly or indirectly) - has the FS write lock. */ -static svn_error_t * -walk_locks(svn_fs_t *fs, - const char *digest_path, - svn_fs_get_locks_callback_t get_locks_func, - void *get_locks_baton, - svn_boolean_t have_write_lock, - apr_pool_t *pool) -{ - walk_locks_baton_t wlb; - - wlb.get_locks_func = get_locks_func; - wlb.get_locks_baton = get_locks_baton; - wlb.fs = fs; - SVN_ERR(walk_digest_files(fs->path, digest_path, locks_walker, &wlb, - have_write_lock, pool)); - return SVN_NO_ERROR; -} - - /* Utility function: verify that a lock can be used. Interesting errors returned from this function: @@ -737,6 +688,35 @@ svn_fs_x__allow_locked_operation(const char *path, return SVN_NO_ERROR; } +/* Helper function called from the lock and unlock code. + UPDATES is a map from "const char *" parent paths to "apr_array_header_t *" + arrays of child paths. For all of the parent paths of PATH this function + adds PATH to the corresponding array of child paths. */ +static void +schedule_index_update(apr_hash_t *updates, + const char *path, + apr_pool_t *scratch_pool) +{ + apr_pool_t *hashpool = apr_hash_pool_get(updates); + const char *parent_path = path; + + while (! svn_fspath__is_root(parent_path, strlen(parent_path))) + { + apr_array_header_t *children; + + parent_path = svn_fspath__dirname(parent_path, scratch_pool); + children = svn_hash_gets(updates, parent_path); + + if (! children) + { + children = apr_array_make(hashpool, 8, sizeof(const char *)); + svn_hash_sets(updates, apr_pstrdup(hashpool, parent_path), children); + } + + APR_ARRAY_PUSH(children, const char *) = path; + } +} + /* The effective arguments for lock_body() below. */ typedef struct lock_baton_t { svn_fs_t *fs; @@ -859,7 +839,6 @@ check_lock(svn_error_t **fs_err, typedef struct lock_info_t { const char *path; - const char *component; svn_lock_t *lock; svn_error_t *fs_err; } lock_info_t; @@ -876,20 +855,20 @@ typedef struct lock_info_t { type, and assumes that the write lock is held. */ static svn_error_t * -lock_body(void *baton, apr_pool_t *pool) +lock_body(void *baton, + apr_pool_t *pool) { lock_baton_t *lb = baton; svn_fs_root_t *root; svn_revnum_t youngest; const char *rev_0_path; - int i, outstanding = 0; + int i; + apr_hash_t *index_updates = apr_hash_make(pool); + apr_hash_index_t *hi; apr_pool_t *iterpool = svn_pool_create(pool); - lb->infos = apr_array_make(lb->result_pool, lb->targets->nelts, - sizeof(lock_info_t)); - /* Until we implement directory locks someday, we only allow locks - on files or non-existent paths. */ + on files. */ /* Use fs->vtable->foo instead of svn_fs_foo to avoid circular library dependencies, which are not portable. */ SVN_ERR(lb->fs->vtable->youngest_rev(&youngest, lb->fs, pool)); @@ -899,35 +878,28 @@ lock_body(void *baton, apr_pool_t *pool) { const svn_sort__item_t *item = &APR_ARRAY_IDX(lb->targets, i, svn_sort__item_t); - const svn_fs_lock_target_t *target = item->value; lock_info_t info; svn_pool_clear(iterpool); info.path = item->key; - SVN_ERR(check_lock(&info.fs_err, info.path, target, lb, root, - youngest, iterpool)); info.lock = NULL; - info.component = NULL; - APR_ARRAY_PUSH(lb->infos, lock_info_t) = info; + info.fs_err = SVN_NO_ERROR; + + SVN_ERR(check_lock(&info.fs_err, info.path, item->value, lb, root, + youngest, iterpool)); + + /* If no error occurred while pre-checking, schedule the index updates for + this path. */ if (!info.fs_err) - ++outstanding; + schedule_index_update(index_updates, info.path, iterpool); + + APR_ARRAY_PUSH(lb->infos, lock_info_t) = info; } rev_0_path = svn_fs_x__path_rev_absolute(lb->fs, 0, pool); - /* Given the paths: - - /foo/bar/f - /foo/bar/g - /zig/x - - we loop through repeatedly. The first pass sees '/' on all paths - and writes the '/' index. The second pass sees '/foo' twice and - writes that index followed by '/zig' and that index. The third - pass sees '/foo/bar' twice and writes that index, and then writes - the lock for '/zig/x'. The fourth pass writes the locks for - '/foo/bar/f' and '/foo/bar/g'. + /* We apply the scheduled index updates before writing the actual locks. Writing indices before locks is correct: if interrupted it leaves indices without locks rather than locks without indices. An @@ -936,91 +908,50 @@ lock_body(void *baton, apr_pool_t *pool) index is inconsistent, svn_fs_x__allow_locked_operation will show locked on the file but unlocked on the parent. */ + for (hi = apr_hash_first(pool, index_updates); hi; hi = apr_hash_next(hi)) + { + const char *path = apr_hash_this_key(hi); + apr_array_header_t *children = apr_hash_this_val(hi); - while (outstanding) + svn_pool_clear(iterpool); + SVN_ERR(add_to_digest(lb->fs->path, children, path, rev_0_path, + iterpool)); + } + + for (i = 0; i < lb->infos->nelts; ++i) { - const char *last_path = NULL; - apr_array_header_t *paths; + struct lock_info_t *info = &APR_ARRAY_IDX(lb->infos, i, + struct lock_info_t); + svn_sort__item_t *item = &APR_ARRAY_IDX(lb->targets, i, svn_sort__item_t); + svn_fs_lock_target_t *target = item->value; svn_pool_clear(iterpool); - paths = apr_array_make(iterpool, 1, sizeof(const char *)); - for (i = 0; i < lb->infos->nelts; ++i) + if (! info->fs_err) { - lock_info_t *info = &APR_ARRAY_IDX(lb->infos, i, lock_info_t); - const svn_sort__item_t *item = &APR_ARRAY_IDX(lb->targets, i, - svn_sort__item_t); - const svn_fs_lock_target_t *target = item->value; - - if (!info->fs_err && !info->lock) - { - if (!info->component) - { - info->component = info->path; - APR_ARRAY_PUSH(paths, const char *) = info->path; - last_path = "/"; - } - else - { - info->component = strchr(info->component + 1, '/'); - if (!info->component) - { - /* The component is a path to lock, this cannot - match a previous path that need to be indexed. */ - if (paths->nelts) - { - SVN_ERR(add_to_digest(lb->fs->path, paths, last_path, - rev_0_path, iterpool)); - apr_array_clear(paths); - last_path = NULL; - } - - info->lock = svn_lock_create(lb->result_pool); - if (target->token) - info->lock->token = target->token; - else - SVN_ERR(svn_fs_x__generate_lock_token( - &(info->lock->token), lb->fs, - lb->result_pool)); - info->lock->path = info->path; - info->lock->owner = lb->fs->access_ctx->username; - info->lock->comment = lb->comment; - info->lock->is_dav_comment = lb->is_dav_comment; - info->lock->creation_date = apr_time_now(); - info->lock->expiration_date = lb->expiration_date; - - info->fs_err = set_lock(lb->fs->path, info->lock, - rev_0_path, iterpool); - --outstanding; - } - else - { - /* The component is a path to an index. */ - apr_size_t len = info->component - info->path; - - if (last_path - && (strncmp(last_path, info->path, len) - || strlen(last_path) != len)) - { - /* No match to the previous paths to index. */ - SVN_ERR(add_to_digest(lb->fs->path, paths, last_path, - rev_0_path, iterpool)); - apr_array_clear(paths); - last_path = NULL; - } - APR_ARRAY_PUSH(paths, const char *) = info->path; - if (!last_path) - last_path = apr_pstrndup(iterpool, info->path, len); - } - } - } - - if (last_path && i == lb->infos->nelts - 1) - SVN_ERR(add_to_digest(lb->fs->path, paths, last_path, - rev_0_path, iterpool)); + info->lock = svn_lock_create(lb->result_pool); + if (target->token) + info->lock->token = apr_pstrdup(lb->result_pool, target->token); + else + SVN_ERR(svn_fs_x__generate_lock_token(&(info->lock->token), lb->fs, + lb->result_pool)); + + /* The INFO->PATH is already allocated in LB->RESULT_POOL as a result + of svn_fspath__canonicalize() (see svn_fs_x__lock()). */ + info->lock->path = info->path; + info->lock->owner = apr_pstrdup(lb->result_pool, + lb->fs->access_ctx->username); + info->lock->comment = apr_pstrdup(lb->result_pool, lb->comment); + info->lock->is_dav_comment = lb->is_dav_comment; + info->lock->creation_date = apr_time_now(); + info->lock->expiration_date = lb->expiration_date; + + info->fs_err = set_lock(lb->fs->path, info->lock, rev_0_path, + iterpool); } } + svn_pool_destroy(iterpool); return SVN_NO_ERROR; } @@ -1061,10 +992,8 @@ check_unlock(svn_error_t **fs_err, typedef struct unlock_info_t { const char *path; - const char *component; svn_error_t *fs_err; svn_boolean_t done; - int components; } unlock_info_t; /* The body of svn_fs_x__unlock(), which see. @@ -1079,18 +1008,18 @@ typedef struct unlock_info_t { type, and assumes that the write lock is held. */ static svn_error_t * -unlock_body(void *baton, apr_pool_t *pool) +unlock_body(void *baton, + apr_pool_t *pool) { unlock_baton_t *ub = baton; svn_fs_root_t *root; svn_revnum_t youngest; const char *rev_0_path; - int i, max_components = 0, outstanding = 0; + int i; + apr_hash_t *indices_updates = apr_hash_make(pool); + apr_hash_index_t *hi; apr_pool_t *iterpool = svn_pool_create(pool); - ub->infos = apr_array_make(ub->result_pool, ub->targets->nelts, - sizeof( unlock_info_t)); - SVN_ERR(ub->fs->vtable->youngest_rev(&youngest, ub->fs, pool)); SVN_ERR(ub->fs->vtable->revision_root(&root, ub->fs, youngest, pool)); @@ -1099,95 +1028,56 @@ unlock_body(void *baton, apr_pool_t *pool) const svn_sort__item_t *item = &APR_ARRAY_IDX(ub->targets, i, svn_sort__item_t); const char *token = item->value; - unlock_info_t info = { 0 }; + unlock_info_t info; svn_pool_clear(iterpool); info.path = item->key; + info.fs_err = SVN_NO_ERROR; + info.done = FALSE; + if (!ub->skip_check) SVN_ERR(check_unlock(&info.fs_err, info.path, token, ub, root, iterpool)); - if (!info.fs_err) - { - const char *s; - info.components = 1; - info.component = info.path; - while((s = strchr(info.component + 1, '/'))) - { - info.component = s; - ++info.components; - } - - if (info.components > max_components) - max_components = info.components; + /* If no error occurred while pre-checking, schedule the index updates for + this path. */ + if (!info.fs_err) + schedule_index_update(indices_updates, info.path, iterpool); - ++outstanding; - } APR_ARRAY_PUSH(ub->infos, unlock_info_t) = info; } rev_0_path = svn_fs_x__path_rev_absolute(ub->fs, 0, pool); - for (i = max_components; i >= 0; --i) + /* Unlike the lock_body(), we need to delete locks *before* we start to + update indices. */ + + for (i = 0; i < ub->infos->nelts; ++i) { - const char *last_path = NULL; - apr_array_header_t *paths; - int j; + struct unlock_info_t *info = &APR_ARRAY_IDX(ub->infos, i, + struct unlock_info_t); svn_pool_clear(iterpool); - paths = apr_array_make(pool, 1, sizeof(const char *)); - for (j = 0; j < ub->infos->nelts; ++j) + if (! info->fs_err) { - unlock_info_t *info = &APR_ARRAY_IDX(ub->infos, j, unlock_info_t); - - if (!info->fs_err && info->path) - { - - if (info->components == i) - { - SVN_ERR(delete_lock(ub->fs->path, info->path, iterpool)); - info->done = TRUE; - } - else if (info->components > i) - { - apr_size_t len = info->component - info->path; - - if (last_path - && strcmp(last_path, "/") - && (strncmp(last_path, info->path, len) - || strlen(last_path) != len)) - { - SVN_ERR(delete_from_digest(ub->fs->path, paths, last_path, - rev_0_path, iterpool)); - apr_array_clear(paths); - last_path = NULL; - } - APR_ARRAY_PUSH(paths, const char *) = info->path; - if (!last_path) - { - if (info->component > info->path) - last_path = apr_pstrndup(pool, info->path, len); - else - last_path = "/"; - } - - if (info->component > info->path) - { - --info->component; - while(info->component[0] != '/') - --info->component; - } - } - } - - if (last_path && j == ub->infos->nelts - 1) - SVN_ERR(delete_from_digest(ub->fs->path, paths, last_path, - rev_0_path, iterpool)); + SVN_ERR(delete_lock(ub->fs->path, info->path, iterpool)); + info->done = TRUE; } } + for (hi = apr_hash_first(pool, indices_updates); hi; hi = apr_hash_next(hi)) + { + const char *path = apr_hash_this_key(hi); + apr_array_header_t *children = apr_hash_this_val(hi); + + svn_pool_clear(iterpool); + SVN_ERR(delete_from_digest(ub->fs->path, children, path, rev_0_path, + iterpool)); + } + + svn_pool_destroy(iterpool); return SVN_NO_ERROR; } @@ -1211,6 +1101,8 @@ unlock_single(svn_fs_t *fs, ub.fs = fs; ub.targets = targets; + ub.infos = apr_array_make(scratch_pool, targets->nelts, + sizeof(struct unlock_info_t)); ub.skip_check = TRUE; ub.result_pool = scratch_pool; @@ -1272,6 +1164,8 @@ svn_fs_x__lock(svn_fs_t *fs, lb.fs = fs; lb.targets = sorted_targets; + lb.infos = apr_array_make(result_pool, sorted_targets->nelts, + sizeof(struct lock_info_t)); lb.comment = comment; lb.is_dav_comment = is_dav_comment; lb.expiration_date = expiration_date; @@ -1366,6 +1260,8 @@ svn_fs_x__unlock(svn_fs_t *fs, ub.fs = fs; ub.targets = sorted_targets; + ub.infos = apr_array_make(result_pool, sorted_targets->nelts, + sizeof(struct unlock_info_t)); ub.skip_check = FALSE; ub.break_lock = break_lock; ub.result_pool = result_pool; diff --git a/subversion/libsvn_fs_x/lock.h b/subversion/libsvn_fs_x/lock.h index 1db5eb71677f..508c1ff0b001 100644 --- a/subversion/libsvn_fs_x/lock.h +++ b/subversion/libsvn_fs_x/lock.h @@ -20,8 +20,8 @@ * ==================================================================== */ -#ifndef SVN_LIBSVN_FS_LOCK_H -#define SVN_LIBSVN_FS_LOCK_H +#ifndef SVN_LIBSVN_FS_X_LOCK_H +#define SVN_LIBSVN_FS_X_LOCK_H #ifdef __cplusplus extern "C" { @@ -113,4 +113,4 @@ svn_fs_x__allow_locked_operation(const char *path, } #endif /* __cplusplus */ -#endif /* SVN_LIBSVN_FS_LOCK_H */ +#endif /* SVN_LIBSVN_FS_X_LOCK_H */ diff --git a/subversion/libsvn_fs_x/low_level.c b/subversion/libsvn_fs_x/low_level.c index 76f3fd2f3872..5c7e3ad26610 100644 --- a/subversion/libsvn_fs_x/low_level.c +++ b/subversion/libsvn_fs_x/low_level.c @@ -1,4 +1,4 @@ -/* low_level.c --- low level r/w access to fs_x file structures +/* low_level.c --- low level r/w access to FSX file structures * * ==================================================================== * Licensed to the Apache Software Foundation (ASF) under one @@ -56,7 +56,6 @@ #define ACTION_ADD "add" #define ACTION_DELETE "delete" #define ACTION_REPLACE "replace" -#define ACTION_RESET "reset" /* True and False flags. */ #define FLAG_TRUE "true" @@ -103,6 +102,19 @@ parse_revnum(svn_revnum_t *rev, return SVN_NO_ERROR; } +/* If ERR is not NULL, wrap it MESSAGE. The latter must have an %ld + * format parameter that will be filled with REV. */ +static svn_error_t * +wrap_footer_error(svn_error_t *err, + const char *message, + svn_revnum_t rev) +{ + if (err) + return svn_error_quick_wrapf(err, message, rev); + + return SVN_NO_ERROR; +} + svn_error_t * svn_fs_x__parse_footer(apr_off_t *l2p_offset, svn_checksum_t **l2p_checksum, @@ -110,6 +122,7 @@ svn_fs_x__parse_footer(apr_off_t *l2p_offset, svn_checksum_t **p2l_checksum, svn_stringbuf_t *footer, svn_revnum_t rev, + apr_off_t footer_offset, apr_pool_t *result_pool) { apr_int64_t val; @@ -118,17 +131,20 @@ svn_fs_x__parse_footer(apr_off_t *l2p_offset, /* Get the L2P offset. */ const char *str = svn_cstring_tokenize(" ", &last_str); if (str == NULL) - return svn_error_create(SVN_ERR_FS_CORRUPT, NULL, - _("Invalid revision footer")); + return svn_error_createf(SVN_ERR_FS_CORRUPT, NULL, + "Invalid r%ld footer", rev); - SVN_ERR(svn_cstring_atoi64(&val, str)); + SVN_ERR(wrap_footer_error(svn_cstring_strtoi64(&val, str, 0, + footer_offset - 1, 10), + "Invalid L2P offset in r%ld footer", + rev)); *l2p_offset = (apr_off_t)val; /* Get the L2P checksum. */ str = svn_cstring_tokenize(" ", &last_str); if (str == NULL) - return svn_error_create(SVN_ERR_FS_CORRUPT, NULL, - _("Invalid revision footer")); + return svn_error_createf(SVN_ERR_FS_CORRUPT, NULL, + "Invalid r%ld footer", rev); SVN_ERR(svn_checksum_parse_hex(l2p_checksum, svn_checksum_md5, str, result_pool)); @@ -136,17 +152,33 @@ svn_fs_x__parse_footer(apr_off_t *l2p_offset, /* Get the P2L offset. */ str = svn_cstring_tokenize(" ", &last_str); if (str == NULL) - return svn_error_create(SVN_ERR_FS_CORRUPT, NULL, - _("Invalid revision footer")); + return svn_error_createf(SVN_ERR_FS_CORRUPT, NULL, + "Invalid r%ld footer", rev); - SVN_ERR(svn_cstring_atoi64(&val, str)); + SVN_ERR(wrap_footer_error(svn_cstring_strtoi64(&val, str, 0, + footer_offset - 1, 10), + "Invalid P2L offset in r%ld footer", + rev)); *p2l_offset = (apr_off_t)val; + /* The P2L indes follows the L2P index */ + if (*p2l_offset <= *l2p_offset) + return svn_error_createf(SVN_ERR_FS_CORRUPT, NULL, + "P2L offset %s must be larger than L2P offset %s" + " in r%ld footer", + apr_psprintf(result_pool, + "%" APR_UINT64_T_HEX_FMT, + (apr_uint64_t)*p2l_offset), + apr_psprintf(result_pool, + "%" APR_UINT64_T_HEX_FMT, + (apr_uint64_t)*l2p_offset), + rev); + /* Get the P2L checksum. */ str = svn_cstring_tokenize(" ", &last_str); if (str == NULL) - return svn_error_create(SVN_ERR_FS_CORRUPT, NULL, - _("Invalid revision footer")); + return svn_error_createf(SVN_ERR_FS_CORRUPT, NULL, + "Invalid r%ld footer", rev); SVN_ERR(svn_checksum_parse_hex(p2l_checksum, svn_checksum_md5, str, result_pool)); @@ -797,14 +829,6 @@ read_change(svn_fs_x__change_t **change_p, change = apr_pcalloc(result_pool, sizeof(*change)); last_str = line->data; - /* Get the node-id of the change. */ - str = svn_cstring_tokenize(" ", &last_str); - if (str == NULL) - return svn_error_create(SVN_ERR_FS_CORRUPT, NULL, - _("Invalid changes line in rev-file")); - - SVN_ERR(svn_fs_x__id_parse(&change->noderev_id, str)); - /* Get the change type. */ str = svn_cstring_tokenize(" ", &last_str); if (str == NULL) @@ -845,10 +869,6 @@ read_change(svn_fs_x__change_t **change_p, { change->change_kind = svn_fs_path_change_replace; } - else if (strcmp(str, ACTION_RESET) == 0) - { - change->change_kind = svn_fs_path_change_reset; - } else { return svn_error_create(SVN_ERR_FS_CORRUPT, NULL, @@ -954,10 +974,10 @@ read_change(svn_fs_x__change_t **change_p, svn_error_t * svn_fs_x__read_changes(apr_array_header_t **changes, svn_stream_t *stream, + int max_count, apr_pool_t *result_pool, apr_pool_t *scratch_pool) { - svn_fs_x__change_t *change; apr_pool_t *iterpool; /* Pre-allocate enough room for most change lists. @@ -970,13 +990,16 @@ svn_fs_x__read_changes(apr_array_header_t **changes, */ *changes = apr_array_make(result_pool, 63, sizeof(svn_fs_x__change_t *)); - SVN_ERR(read_change(&change, stream, result_pool, scratch_pool)); iterpool = svn_pool_create(scratch_pool); - while (change) + for (; max_count > 0; --max_count) { - APR_ARRAY_PUSH(*changes, svn_fs_x__change_t*) = change; - SVN_ERR(read_change(&change, stream, result_pool, iterpool)); + svn_fs_x__change_t *change; svn_pool_clear(iterpool); + SVN_ERR(read_change(&change, stream, result_pool, iterpool)); + if (!change) + break; + + APR_ARRAY_PUSH(*changes, svn_fs_x__change_t*) = change; } svn_pool_destroy(iterpool); @@ -1016,7 +1039,6 @@ write_change_entry(svn_stream_t *stream, svn_fs_x__change_t *change, apr_pool_t *scratch_pool) { - const char *idstr; const char *change_string = NULL; const char *kind_string = ""; svn_stringbuf_t *buf; @@ -1036,17 +1058,12 @@ write_change_entry(svn_stream_t *stream, case svn_fs_path_change_replace: change_string = ACTION_REPLACE; break; - case svn_fs_path_change_reset: - change_string = ACTION_RESET; - break; default: return svn_error_createf(SVN_ERR_FS_CORRUPT, NULL, _("Invalid change type %d"), change->change_kind); } - idstr = svn_fs_x__id_unparse(&change->noderev_id, scratch_pool)->data; - SVN_ERR_ASSERT(change->node_kind == svn_node_dir || change->node_kind == svn_node_file); kind_string = apr_psprintf(scratch_pool, "-%s", @@ -1054,8 +1071,8 @@ write_change_entry(svn_stream_t *stream, ? SVN_FS_X__KIND_DIR : SVN_FS_X__KIND_FILE); - buf = svn_stringbuf_createf(scratch_pool, "%s %s%s %s %s %s %s\n", - idstr, change_string, kind_string, + buf = svn_stringbuf_createf(scratch_pool, "%s%s %s %s %s %s\n", + change_string, kind_string, change->text_mod ? FLAG_TRUE : FLAG_FALSE, change->prop_mod ? FLAG_TRUE : FLAG_FALSE, change->mergeinfo_mod == svn_tristate_true @@ -1121,3 +1138,107 @@ svn_fs_x__write_changes(svn_stream_t *stream, return SVN_NO_ERROR; } +svn_error_t * +svn_fs_x__parse_properties(apr_hash_t **properties, + const svn_string_t *content, + apr_pool_t *result_pool) +{ + const apr_byte_t *p = (const apr_byte_t *)content->data; + const apr_byte_t *end = p + content->len; + apr_uint64_t count; + + *properties = apr_hash_make(result_pool); + + /* Extract the number of properties we are expected to read. */ + p = svn__decode_uint(&count, p, end); + + /* Read all the properties we find. + Because prop-name and prop-value are nicely NUL-terminated + sub-strings of CONTENT, we can simply reference them there. + I.e. there is no need to copy them around. + */ + while (p < end) + { + apr_uint64_t value_len; + svn_string_t *value; + + const char *key = (const char *)p; + + /* Note that this may never overflow / segfault because + CONTENT itself is NUL-terminated. */ + apr_size_t key_len = strlen(key); + p += key_len + 1; + if (key[key_len]) + return svn_error_createf(SVN_ERR_FS_CORRUPT_PROPLIST, NULL, + "Property name not NUL terminated"); + + if (p >= end) + return svn_error_createf(SVN_ERR_FS_CORRUPT_PROPLIST, NULL, + "Property value missing"); + p = svn__decode_uint(&value_len, p, end); + if (value_len >= (end - p)) + return svn_error_createf(SVN_ERR_FS_CORRUPT_PROPLIST, NULL, + "Property value too long"); + + value = apr_pcalloc(result_pool, sizeof(*value)); + value->data = (const char *)p; + value->len = (apr_size_t)value_len; + if (p[value->len]) + return svn_error_createf(SVN_ERR_FS_CORRUPT_PROPLIST, NULL, + "Property value not NUL terminated"); + + p += value->len + 1; + + apr_hash_set(*properties, key, key_len, value); + } + + /* Check that we read the expected number of properties. */ + if ((apr_uint64_t)apr_hash_count(*properties) != count) + return svn_error_createf(SVN_ERR_FS_CORRUPT_PROPLIST, NULL, + "Property count mismatch"); + + return SVN_NO_ERROR; +} + +svn_error_t * +svn_fs_x__write_properties(svn_stream_t *stream, + apr_hash_t *proplist, + apr_pool_t *scratch_pool) +{ + apr_byte_t buffer[SVN__MAX_ENCODED_UINT_LEN]; + apr_size_t len; + apr_hash_index_t *hi; + + /* Write the number of properties in this list. */ + len = svn__encode_uint(buffer, apr_hash_count(proplist)) - buffer; + SVN_ERR(svn_stream_write(stream, (const char *)buffer, &len)); + + /* Serialize each property as follows: + <Prop-name> <NUL> + <Value-len> <Prop-value> <NUL> + */ + for (hi = apr_hash_first(scratch_pool, proplist); + hi; + hi = apr_hash_next(hi)) + { + const char *key; + apr_size_t key_len; + svn_string_t *value; + apr_hash_this(hi, (const void **)&key, (apr_ssize_t *)&key_len, + (void **)&value); + + /* Include the terminating NUL. */ + ++key_len; + SVN_ERR(svn_stream_write(stream, key, &key_len)); + + len = svn__encode_uint(buffer, value->len) - buffer; + SVN_ERR(svn_stream_write(stream, (const char *)buffer, &len)); + SVN_ERR(svn_stream_write(stream, value->data, &value->len)); + + /* Terminate with NUL. */ + len = 1; + SVN_ERR(svn_stream_write(stream, "", &len)); + } + + return SVN_NO_ERROR; +} diff --git a/subversion/libsvn_fs_x/low_level.h b/subversion/libsvn_fs_x/low_level.h index e4fdf05a8d4b..d59618d1698b 100644 --- a/subversion/libsvn_fs_x/low_level.h +++ b/subversion/libsvn_fs_x/low_level.h @@ -1,4 +1,4 @@ -/* low_level.c --- low level r/w access to fs_x file structures +/* low_level.c --- low level r/w access to FSX file structures * * ==================================================================== * Licensed to the Apache Software Foundation (ASF) under one @@ -20,8 +20,8 @@ * ==================================================================== */ -#ifndef SVN_LIBSVN_FS__LOW_LEVEL_H -#define SVN_LIBSVN_FS__LOW_LEVEL_H +#ifndef SVN_LIBSVN_FS_X_LOW_LEVEL_H +#define SVN_LIBSVN_FS_X_LOW_LEVEL_H #include "svn_fs.h" @@ -50,6 +50,8 @@ extern "C" { * *P2L_OFFSET, respectively. Also, return the expected checksums in * in *L2P_CHECKSUM and *P2L_CHECKSUM. * + * FOOTER_OFFSET is used for validation. + * * Note that REV is only used to construct nicer error objects that * mention this revision. Allocate the checksums in RESULT_POOL. */ @@ -60,6 +62,7 @@ svn_fs_x__parse_footer(apr_off_t *l2p_offset, svn_checksum_t **p2l_checksum, svn_stringbuf_t *footer, svn_revnum_t rev, + apr_off_t footer_offset, apr_pool_t *result_pool); /* Given the offset of the L2P index data in L2P_OFFSET, the content @@ -167,15 +170,17 @@ svn_fs_x__write_rep_header(svn_fs_x__rep_header_t *header, svn_stream_t *stream, apr_pool_t *scratch_pool); -/* Read all the changes from STREAM and store them in *CHANGES, - allocated in RESULT_POOL. Do temporary allocations in SCRATCH_POOL. */ +/* Read up to MAX_COUNT of the changes from STREAM and store them in + *CHANGES, allocated in RESULT_POOL. Do temporary allocations in + SCRATCH_POOL. */ svn_error_t * svn_fs_x__read_changes(apr_array_header_t **changes, svn_stream_t *stream, + int max_count, apr_pool_t *result_pool, apr_pool_t *scratch_pool); -/* Callback function used by svn_fs_fs__read_changes_incrementally(), +/* Callback function used by svn_fs_x__read_changes_incrementally(), * asking the receiver to process to process CHANGE using BATON. CHANGE * and SCRATCH_POOL will not be valid beyond the current callback invocation. */ @@ -207,8 +212,25 @@ svn_fs_x__write_changes(svn_stream_t *stream, svn_boolean_t terminate_list, apr_pool_t *scratch_pool); +/* Parse the property list serialized in CONTENT and return it in + *PROPERTIES, allocated from RESULT_POOL. CONTENT must remain + valid at least until the next cleanup of RESULT_POOL. + */ +svn_error_t * +svn_fs_x__parse_properties(apr_hash_t **properties, + const svn_string_t *content, + apr_pool_t *result_pool); + +/* Write the property list PROPLIST to STREAM in serialized format. + Use SCRATCH_POOL for temporary allocations. + */ +svn_error_t * +svn_fs_x__write_properties(svn_stream_t *stream, + apr_hash_t *proplist, + apr_pool_t *scratch_pool); + #ifdef __cplusplus } #endif /* __cplusplus */ -#endif /* SVN_LIBSVN_FS__LOW_LEVEL_H */ +#endif /* SVN_LIBSVN_FS_X_LOW_LEVEL_H */ diff --git a/subversion/libsvn_fs_x/noderevs.c b/subversion/libsvn_fs_x/noderevs.c index 60c6029966ca..8e6cf7432268 100644 --- a/subversion/libsvn_fs_x/noderevs.c +++ b/subversion/libsvn_fs_x/noderevs.c @@ -361,7 +361,7 @@ svn_error_t * svn_fs_x__noderevs_get(svn_fs_x__noderev_t **noderev_p, const svn_fs_x__noderevs_t *container, apr_size_t idx, - apr_pool_t *pool) + apr_pool_t *result_pool) { svn_fs_x__noderev_t *noderev; binary_noderev_t *binary_noderev; @@ -373,14 +373,14 @@ svn_fs_x__noderevs_get(svn_fs_x__noderev_t **noderev_p, /* validate index */ if (idx >= (apr_size_t)container->noderevs->nelts) return svn_error_createf(SVN_ERR_FS_CONTAINER_INDEX, NULL, - apr_psprintf(pool, + apr_psprintf(result_pool, _("Node revision index %%%s" " exceeds container size %%d"), APR_SIZE_T_FMT), idx, container->noderevs->nelts); /* allocate result struct and fill it field by field */ - noderev = apr_pcalloc(pool, sizeof(*noderev)); + noderev = apr_pcalloc(result_pool, sizeof(*noderev)); binary_noderev = &APR_ARRAY_IDX(container->noderevs, idx, binary_noderev_t); noderev->kind = (svn_node_kind_t)(binary_noderev->flags & NODEREV_KIND_MASK); @@ -398,7 +398,7 @@ svn_fs_x__noderevs_get(svn_fs_x__noderev_t **noderev_p, = svn_fs_x__string_table_get(container->paths, binary_noderev->copyfrom_path, NULL, - pool); + result_pool); noderev->copyfrom_rev = binary_noderev->copyfrom_rev; } else @@ -413,7 +413,7 @@ svn_fs_x__noderevs_get(svn_fs_x__noderev_t **noderev_p, = svn_fs_x__string_table_get(container->paths, binary_noderev->copyroot_path, NULL, - pool); + result_pool); noderev->copyroot_rev = binary_noderev->copyroot_rev; } else @@ -425,16 +425,16 @@ svn_fs_x__noderevs_get(svn_fs_x__noderev_t **noderev_p, noderev->predecessor_count = binary_noderev->predecessor_count; SVN_ERR(get_representation(&noderev->prop_rep, container->reps, - binary_noderev->prop_rep, pool)); + binary_noderev->prop_rep, result_pool)); SVN_ERR(get_representation(&noderev->data_rep, container->reps, - binary_noderev->data_rep, pool)); + binary_noderev->data_rep, result_pool)); if (binary_noderev->flags & NODEREV_HAS_CPATH) noderev->created_path = svn_fs_x__string_table_get(container->paths, binary_noderev->created_path, NULL, - pool); + result_pool); noderev->mergeinfo_count = binary_noderev->mergeinfo_count; @@ -578,15 +578,15 @@ svn_fs_x__write_noderevs_container(svn_stream_t *stream, return SVN_NO_ERROR; } -/* Allocate a svn_fs_x__representation_t array in POOL and return it in - * REPS_P. Deserialize the data in REP_STREAM and DIGEST_STREAM and store +/* Allocate a svn_fs_x__representation_t array in RESULT_POOL and return it + * in REPS_P. Deserialize the data in REP_STREAM and DIGEST_STREAM and store * the resulting representations into the *REPS_P. */ static svn_error_t * read_reps(apr_array_header_t **reps_p, svn_packed__int_stream_t *rep_stream, svn_packed__byte_stream_t *digest_stream, - apr_pool_t *pool) + apr_pool_t *result_pool) { apr_size_t i; apr_size_t len; @@ -595,7 +595,8 @@ read_reps(apr_array_header_t **reps_p, apr_size_t count = svn_packed__int_count(svn_packed__first_int_substream(rep_stream)); apr_array_header_t *reps - = apr_array_make(pool, (int)count, sizeof(svn_fs_x__representation_t)); + = apr_array_make(result_pool, (int)count, + sizeof(svn_fs_x__representation_t)); for (i = 0; i < count; ++i) { @@ -613,7 +614,7 @@ read_reps(apr_array_header_t **reps_p, bytes = svn_packed__get_bytes(digest_stream, &len); if (len != sizeof(rep.md5_digest)) return svn_error_createf(SVN_ERR_FS_CONTAINER_INDEX, NULL, - apr_psprintf(pool, + apr_psprintf(result_pool, _("Unexpected MD5" " digest size %%%s"), APR_SIZE_T_FMT), @@ -625,7 +626,7 @@ read_reps(apr_array_header_t **reps_p, bytes = svn_packed__get_bytes(digest_stream, &len); if (len != sizeof(rep.sha1_digest)) return svn_error_createf(SVN_ERR_FS_CONTAINER_INDEX, NULL, - apr_psprintf(pool, + apr_psprintf(result_pool, _("Unexpected SHA1" " digest size %%%s"), APR_SIZE_T_FMT), @@ -766,15 +767,15 @@ svn_error_t * svn_fs_x__deserialize_noderevs_container(void **out, void *data, apr_size_t data_len, - apr_pool_t *pool) + apr_pool_t *result_pool) { svn_fs_x__noderevs_t *noderevs = (svn_fs_x__noderevs_t *)data; /* de-serialize sub-structures */ svn_fs_x__deserialize_string_table(noderevs, &noderevs->paths); - svn_fs_x__deserialize_apr_array(noderevs, &noderevs->ids, pool); - svn_fs_x__deserialize_apr_array(noderevs, &noderevs->reps, pool); - svn_fs_x__deserialize_apr_array(noderevs, &noderevs->noderevs, pool); + svn_fs_x__deserialize_apr_array(noderevs, &noderevs->ids, result_pool); + svn_fs_x__deserialize_apr_array(noderevs, &noderevs->reps, result_pool); + svn_fs_x__deserialize_apr_array(noderevs, &noderevs->noderevs, result_pool); /* done */ *out = noderevs; diff --git a/subversion/libsvn_fs_x/noderevs.h b/subversion/libsvn_fs_x/noderevs.h index f9b79dcf2be1..72b2ef84d239 100644 --- a/subversion/libsvn_fs_x/noderevs.h +++ b/subversion/libsvn_fs_x/noderevs.h @@ -20,8 +20,8 @@ * ==================================================================== */ -#ifndef SVN_LIBSVN_FS__NODEREVS_H -#define SVN_LIBSVN_FS__NODEREVS_H +#ifndef SVN_LIBSVN_FS_X_NODEREVS_H +#define SVN_LIBSVN_FS_X_NODEREVS_H #include "svn_io.h" #include "fs.h" @@ -76,7 +76,7 @@ svn_error_t * svn_fs_x__noderevs_get(svn_fs_x__noderev_t **noderev_p, const svn_fs_x__noderevs_t *container, apr_size_t idx, - apr_pool_t *pool); + apr_pool_t *result_pool); /* I/O interface. */ @@ -114,7 +114,7 @@ svn_error_t * svn_fs_x__deserialize_noderevs_container(void **out, void *data, apr_size_t data_len, - apr_pool_t *pool); + apr_pool_t *result_pool); /* Implements svn_cache__partial_getter_func_t for svn_fs_x__noderevs_t, * setting *OUT to the svn_fs_x__noderev_t selected by the apr_uint32_t index diff --git a/subversion/libsvn_fs_x/pack.c b/subversion/libsvn_fs_x/pack.c index cdbb98029322..095b04ec8297 100644 --- a/subversion/libsvn_fs_x/pack.c +++ b/subversion/libsvn_fs_x/pack.c @@ -106,9 +106,6 @@ typedef struct path_order_t /* when this change happened */ svn_revnum_t revision; - /* this is a directory node */ - svn_boolean_t is_dir; - /* length of the expanded representation content */ apr_int64_t expanded_size; @@ -219,7 +216,7 @@ typedef struct pack_context_t * to NULL that we already processed. */ apr_array_header_t *reps; - /* array of int, marking for each revision, the which offset their items + /* array of int, marking for each revision, at which offset their items * begin in REPS. Will be filled in phase 2 and be cleared after * each revision range. */ apr_array_header_t *rev_offsets; @@ -247,6 +244,7 @@ initialize_pack_context(pack_context_t *context, const char *shard_dir, svn_revnum_t shard_rev, int max_items, + svn_fs_x__batch_fsync_t *batch, svn_cancel_func_t cancel_func, void *cancel_baton, apr_pool_t *pool) @@ -275,9 +273,9 @@ initialize_pack_context(pack_context_t *context, context->pack_file_dir = pack_file_dir; context->pack_file_path = svn_dirent_join(pack_file_dir, PATH_PACKED, pool); - SVN_ERR(svn_io_file_open(&context->pack_file, context->pack_file_path, - APR_WRITE | APR_BUFFERED | APR_BINARY | APR_EXCL - | APR_CREATE, APR_OS_DEFAULT, pool)); + + SVN_ERR(svn_fs_x__batch_fsync_open_file(&context->pack_file, batch, + context->pack_file_path, pool)); /* Proto index files */ SVN_ERR(svn_fs_x__l2p_proto_index_open( @@ -346,6 +344,7 @@ reset_pack_context(pack_context_t *context, SVN_ERR(svn_io_file_trunc(context->reps_file, 0, scratch_pool)); svn_pool_clear(context->info_pool); + context->paths = svn_prefix_tree__create(context->info_pool); return SVN_NO_ERROR; } @@ -382,8 +381,6 @@ close_pack_context(pack_context_t *context, SVN_ERR(svn_io_remove_file2(proto_l2p_index_path, FALSE, scratch_pool)); SVN_ERR(svn_io_remove_file2(proto_p2l_index_path, FALSE, scratch_pool)); - SVN_ERR(svn_io_file_close(context->pack_file, scratch_pool)); - return SVN_NO_ERROR; } @@ -395,7 +392,7 @@ static svn_error_t * copy_file_data(pack_context_t *context, apr_file_t *dest, apr_file_t *source, - apr_off_t size, + svn_filesize_t size, apr_pool_t *scratch_pool) { /* most non-representation items will be small. Minimize the buffer @@ -478,14 +475,16 @@ copy_item_to_temp(pack_context_t *context, svn_fs_x__p2l_entry_t *entry, apr_pool_t *scratch_pool) { + apr_file_t *file; svn_fs_x__p2l_entry_t *new_entry = svn_fs_x__p2l_entry_dup(entry, context->info_pool); - SVN_ERR(svn_fs_x__get_file_offset(&new_entry->offset, temp_file, - scratch_pool)); + SVN_ERR(svn_io_file_get_offset(&new_entry->offset, temp_file, + scratch_pool)); APR_ARRAY_PUSH(entries, svn_fs_x__p2l_entry_t *) = new_entry; - SVN_ERR(copy_file_data(context, temp_file, rev_file->file, entry->size, + SVN_ERR(svn_fs_x__rev_file_get(&file, rev_file)); + SVN_ERR(copy_file_data(context, temp_file, file, entry->size, scratch_pool)); return SVN_NO_ERROR; @@ -567,17 +566,20 @@ copy_rep_to_temp(pack_context_t *context, apr_pool_t *scratch_pool) { svn_fs_x__rep_header_t *rep_header; + svn_stream_t *stream; + apr_file_t *file; apr_off_t source_offset = entry->offset; /* create a copy of ENTRY, make it point to the copy destination and * store it in CONTEXT */ entry = svn_fs_x__p2l_entry_dup(entry, context->info_pool); - SVN_ERR(svn_fs_x__get_file_offset(&entry->offset, context->reps_file, - scratch_pool)); + SVN_ERR(svn_io_file_get_offset(&entry->offset, context->reps_file, + scratch_pool)); add_item_rep_mapping(context, entry); /* read & parse the representation header */ - SVN_ERR(svn_fs_x__read_rep_header(&rep_header, rev_file->stream, + SVN_ERR(svn_fs_x__rev_file_stream(&stream, rev_file)); + SVN_ERR(svn_fs_x__read_rep_header(&rep_header, stream, scratch_pool, scratch_pool)); /* if the representation is a delta against some other rep, link the two */ @@ -594,10 +596,10 @@ copy_rep_to_temp(pack_context_t *context, } /* copy the whole rep (including header!) to our temp file */ - SVN_ERR(svn_io_file_seek(rev_file->file, APR_SET, &source_offset, - scratch_pool)); - SVN_ERR(copy_file_data(context, context->reps_file, rev_file->file, - entry->size, scratch_pool)); + SVN_ERR(svn_fs_x__rev_file_seek(rev_file, NULL, source_offset)); + SVN_ERR(svn_fs_x__rev_file_get(&file, rev_file)); + SVN_ERR(copy_file_data(context, context->reps_file, file, entry->size, + scratch_pool)); return SVN_NO_ERROR; } @@ -616,9 +618,6 @@ compare_dir_entries(const svn_sort__item_t *a, const svn_fs_dirent_t *lhs = (const svn_fs_dirent_t *) a->value; const svn_fs_dirent_t *rhs = (const svn_fs_dirent_t *) b->value; - if (lhs->kind != rhs->kind) - return lhs->kind == svn_node_dir ? -1 : 1; - return strcmp(lhs->name, rhs->name); } @@ -642,7 +641,7 @@ svn_fs_x__order_dir_entries(svn_fs_t *fs, return result; } -/* Return a duplicate of the the ORIGINAL path and with special sub-strins +/* Return a duplicate of the ORIGINAL path and with special sub-strings * (e.g. "trunk") modified in such a way that have a lower lexicographic * value than any other "normal" file name. */ @@ -687,25 +686,28 @@ copy_node_to_temp(pack_context_t *context, path_order_t *path_order = apr_pcalloc(context->info_pool, sizeof(*path_order)); svn_fs_x__noderev_t *noderev; + svn_stream_t *stream; + apr_file_t *file; const char *sort_path; apr_off_t source_offset = entry->offset; /* read & parse noderev */ - SVN_ERR(svn_fs_x__read_noderev(&noderev, rev_file->stream, scratch_pool, + SVN_ERR(svn_fs_x__rev_file_stream(&stream, rev_file)); + SVN_ERR(svn_fs_x__read_noderev(&noderev, stream, scratch_pool, scratch_pool)); /* create a copy of ENTRY, make it point to the copy destination and * store it in CONTEXT */ entry = svn_fs_x__p2l_entry_dup(entry, context->info_pool); - SVN_ERR(svn_fs_x__get_file_offset(&entry->offset, context->reps_file, - scratch_pool)); + SVN_ERR(svn_io_file_get_offset(&entry->offset, context->reps_file, + scratch_pool)); add_item_rep_mapping(context, entry); /* copy the noderev to our temp file */ - SVN_ERR(svn_io_file_seek(rev_file->file, APR_SET, &source_offset, - scratch_pool)); - SVN_ERR(copy_file_data(context, context->reps_file, rev_file->file, - entry->size, scratch_pool)); + SVN_ERR(svn_fs_x__rev_file_seek(rev_file, NULL, source_offset)); + SVN_ERR(svn_fs_x__rev_file_get(&file, rev_file)); + SVN_ERR(copy_file_data(context, context->reps_file, file, entry->size, + scratch_pool)); /* if the node has a data representation, make that the node's "base". * This will (often) cause the noderev to be placed right in front of @@ -732,7 +734,6 @@ copy_node_to_temp(pack_context_t *context, path_order->path = svn_prefix_string__create(context->paths, sort_path); path_order->node_id = noderev->node_id; path_order->revision = svn_fs_x__get_revnum(noderev->noderev_id.change_set); - path_order->is_dir = noderev->kind == svn_node_dir; path_order->noderev_id = noderev->noderev_id; APR_ARRAY_PUSH(context->path_order, path_order_t *) = path_order; @@ -776,13 +777,8 @@ compare_path_order(const path_order_t * const * lhs_p, const path_order_t * lhs = *lhs_p; const path_order_t * rhs = *rhs_p; - /* cluster all directories */ - int diff = rhs->is_dir - lhs->is_dir; - if (diff) - return diff; - /* lexicographic order on path and node (i.e. latest first) */ - diff = svn_prefix_string__compare(lhs->path, rhs->path); + int diff = svn_prefix_string__compare(lhs->path, rhs->path); if (diff) return diff; @@ -826,7 +822,7 @@ sort_reps(pack_context_t *context) /* Return the remaining unused bytes in the current block in CONTEXT's * pack file. */ -static apr_ssize_t +static apr_off_t get_block_left(pack_context_t *context) { svn_fs_x__data_t *ffd = context->fs->fsap_data; @@ -1234,7 +1230,7 @@ write_reps_containers(pack_context_t *context, = apr_array_make(scratch_pool, 64, sizeof(svn_fs_x__id_t)); svn_fs_x__revision_file_t *file; - SVN_ERR(svn_fs_x__wrap_temp_rev_file(&file, context->fs, temp_file, + SVN_ERR(svn_fs_x__rev_file_wrap_temp(&file, context->fs, temp_file, scratch_pool)); /* copy all items in strict order */ @@ -1608,8 +1604,8 @@ write_changes_containers(pack_context_t *context, * the container */ SVN_ERR(svn_io_file_seek(temp_file, APR_SET, &entry->offset, iterpool)); - SVN_ERR(svn_fs_x__read_changes(&changes, temp_stream, scratch_pool, - iterpool)); + SVN_ERR(svn_fs_x__read_changes(&changes, temp_stream, INT_MAX, + scratch_pool, iterpool)); SVN_ERR(svn_fs_x__changes_append_list(&list_index, container, changes)); SVN_ERR_ASSERT(list_index == sub_items->nelts); block_left -= estimated_size; @@ -1723,18 +1719,19 @@ pack_range(pack_context_t *context, { apr_off_t offset = 0; svn_fs_x__revision_file_t *rev_file; + svn_fs_x__index_info_t l2p_index_info; /* Get the rev file dimensions (mainly index locations). */ - SVN_ERR(svn_fs_x__open_pack_or_rev_file(&rev_file, context->fs, - revision, revpool, iterpool)); - SVN_ERR(svn_fs_x__auto_read_footer(rev_file)); + SVN_ERR(svn_fs_x__rev_file_init(&rev_file, context->fs, revision, + revpool)); + SVN_ERR(svn_fs_x__rev_file_l2p_info(&l2p_index_info, rev_file)); /* store the indirect array index */ APR_ARRAY_PUSH(context->rev_offsets, int) = context->reps->nelts; /* read the phys-to-log index file until we covered the whole rev file. * That index contains enough info to build both target indexes from it. */ - while (offset < rev_file->l2p_offset) + while (offset < l2p_index_info.start) { /* read one cluster */ int i; @@ -1758,10 +1755,9 @@ pack_range(pack_context_t *context, /* process entry while inside the rev file */ offset = entry->offset; - if (offset < rev_file->l2p_offset) + if (offset < l2p_index_info.start) { - SVN_ERR(svn_io_file_seek(rev_file->file, APR_SET, &offset, - iterpool)); + SVN_ERR(svn_fs_x__rev_file_seek(rev_file, NULL, offset)); if (entry->type == SVN_FS_X__ITEM_TYPE_CHANGES) SVN_ERR(copy_item_to_temp(context, @@ -1843,21 +1839,19 @@ append_revision(pack_context_t *context, apr_off_t offset = 0; apr_pool_t *iterpool = svn_pool_create(scratch_pool); svn_fs_x__revision_file_t *rev_file; - apr_finfo_t finfo; - - /* Get the size of the file. */ - const char *path = svn_dirent_join(context->shard_dir, - apr_psprintf(iterpool, "%ld", - context->start_rev), - scratch_pool); - SVN_ERR(svn_io_stat(&finfo, path, APR_FINFO_SIZE, scratch_pool)); - - /* Copy all the bits from the rev file to the end of the pack file. */ - SVN_ERR(svn_fs_x__open_pack_or_rev_file(&rev_file, context->fs, - context->start_rev, scratch_pool, - iterpool)); - SVN_ERR(copy_file_data(context, context->pack_file, rev_file->file, - finfo.size, iterpool)); + apr_file_t *file; + svn_filesize_t revdata_size; + + /* Copy all non-index contents the rev file to the end of the pack file. */ + SVN_ERR(svn_fs_x__rev_file_init(&rev_file, context->fs, context->start_rev, + scratch_pool)); + SVN_ERR(svn_fs_x__rev_file_data_size(&revdata_size, rev_file)); + + SVN_ERR(svn_fs_x__rev_file_get(&file, rev_file)); + SVN_ERR(svn_io_file_aligned_seek(file, ffd->block_size, NULL, 0, + iterpool)); + SVN_ERR(copy_file_data(context, context->pack_file, file, revdata_size, + iterpool)); /* mark the start of a new revision */ SVN_ERR(svn_fs_x__l2p_proto_index_add_revision(context->proto_l2p_index, @@ -1865,7 +1859,7 @@ append_revision(pack_context_t *context, /* read the phys-to-log index file until we covered the whole rev file. * That index contains enough info to build both target indexes from it. */ - while (offset < finfo.size) + while (offset < revdata_size) { /* read one cluster */ int i; @@ -1887,7 +1881,7 @@ append_revision(pack_context_t *context, /* process entry while inside the rev file */ offset = entry->offset; - if (offset < finfo.size) + if (offset < revdata_size) { /* there should be true containers */ SVN_ERR_ASSERT(entry->item_count == 1); @@ -1906,7 +1900,7 @@ append_revision(pack_context_t *context, } svn_pool_destroy(iterpool); - context->pack_offset += finfo.size; + context->pack_offset += revdata_size; return SVN_NO_ERROR; } @@ -1917,6 +1911,7 @@ append_revision(pack_context_t *context, * SHARD_DIR into the PACK_FILE_DIR, using SCRATCH_POOL for temporary * allocations. Limit the extra memory consumption to MAX_MEM bytes. * CANCEL_FUNC and CANCEL_BATON are what you think they are. + * Schedule necessary fsync calls in BATCH. */ static svn_error_t * pack_log_addressed(svn_fs_t *fs, @@ -1924,6 +1919,7 @@ pack_log_addressed(svn_fs_t *fs, const char *shard_dir, svn_revnum_t shard_rev, apr_size_t max_mem, + svn_fs_x__batch_fsync_t *batch, svn_cancel_func_t cancel_func, void *cancel_baton, apr_pool_t *scratch_pool) @@ -1950,7 +1946,7 @@ pack_log_addressed(svn_fs_t *fs, /* set up a pack context */ SVN_ERR(initialize_pack_context(&context, fs, pack_file_dir, shard_dir, - shard_rev, max_items, cancel_func, + shard_rev, max_items, batch, cancel_func, cancel_baton, scratch_pool)); /* phase 1: determine the size of the revisions to pack */ @@ -1960,8 +1956,10 @@ pack_log_addressed(svn_fs_t *fs, /* pack revisions in ranges that don't exceed MAX_MEM */ for (i = 0; i < max_ids->nelts; ++i) - if (APR_ARRAY_IDX(max_ids, i, apr_uint64_t) + item_count <= max_items) + if ( APR_ARRAY_IDX(max_ids, i, apr_uint64_t) + <= (apr_uint64_t)max_items - item_count) { + item_count += APR_ARRAY_IDX(max_ids, i, apr_uint64_t); context.end_rev++; } else @@ -2003,77 +2001,11 @@ pack_log_addressed(svn_fs_t *fs, return SVN_NO_ERROR; } -/* Given REV in FS, set *REV_OFFSET to REV's offset in the packed file. - Use SCRATCH_POOL for temporary allocations. */ -svn_error_t * -svn_fs_x__get_packed_offset(apr_off_t *rev_offset, - svn_fs_t *fs, - svn_revnum_t rev, - apr_pool_t *scratch_pool) -{ - svn_fs_x__data_t *ffd = fs->fsap_data; - svn_stream_t *manifest_stream; - svn_boolean_t is_cached; - svn_revnum_t shard; - apr_int64_t shard_pos; - apr_array_header_t *manifest; - apr_pool_t *iterpool; - - shard = rev / ffd->max_files_per_dir; - - /* position of the shard within the manifest */ - shard_pos = rev % ffd->max_files_per_dir; - - /* fetch exactly that element into *rev_offset, if the manifest is found - in the cache */ - SVN_ERR(svn_cache__get_partial((void **) rev_offset, &is_cached, - ffd->packed_offset_cache, &shard, - svn_fs_x__get_sharded_offset, &shard_pos, - scratch_pool)); - - if (is_cached) - return SVN_NO_ERROR; - - /* Open the manifest file. */ - SVN_ERR(svn_stream_open_readonly(&manifest_stream, - svn_fs_x__path_rev_packed(fs, rev, PATH_MANIFEST, - scratch_pool), - scratch_pool, scratch_pool)); - - /* While we're here, let's just read the entire manifest file into an array, - so we can cache the entire thing. */ - iterpool = svn_pool_create(scratch_pool); - manifest = apr_array_make(scratch_pool, ffd->max_files_per_dir, - sizeof(apr_off_t)); - while (1) - { - svn_boolean_t eof; - apr_int64_t val; - - svn_pool_clear(iterpool); - SVN_ERR(svn_fs_x__read_number_from_stream(&val, &eof, manifest_stream, - iterpool)); - if (eof) - break; - - APR_ARRAY_PUSH(manifest, apr_off_t) = (apr_off_t)val; - } - svn_pool_destroy(iterpool); - - *rev_offset = APR_ARRAY_IDX(manifest, rev % ffd->max_files_per_dir, - apr_off_t); - - /* Close up shop and cache the array. */ - SVN_ERR(svn_stream_close(manifest_stream)); - return svn_cache__set(ffd->packed_offset_cache, &shard, manifest, - scratch_pool); -} - /* In filesystem FS, pack the revision SHARD containing exactly * MAX_FILES_PER_DIR revisions from SHARD_PATH into the PACK_FILE_DIR, * using SCRATCH_POOL for temporary allocations. Try to limit the amount of * temporary memory needed to MAX_MEM bytes. CANCEL_FUNC and CANCEL_BATON - * are what you think they are. + * are what you think they are. Schedule necessary fsync calls in BATCH. * * If for some reason we detect a partial packing already performed, we * remove the pack file and start again. @@ -2087,6 +2019,7 @@ pack_rev_shard(svn_fs_t *fs, apr_int64_t shard, int max_files_per_dir, apr_size_t max_mem, + svn_fs_x__batch_fsync_t *batch, svn_cancel_func_t cancel_func, void *cancel_baton, apr_pool_t *scratch_pool) @@ -2103,10 +2036,11 @@ pack_rev_shard(svn_fs_t *fs, /* Create the new directory and pack file. */ SVN_ERR(svn_io_dir_make(pack_file_dir, APR_OS_DEFAULT, scratch_pool)); + SVN_ERR(svn_fs_x__batch_fsync_new_path(batch, pack_file_dir, scratch_pool)); /* Index information files */ SVN_ERR(pack_log_addressed(fs, pack_file_dir, shard_path, shard_rev, - max_mem, cancel_func, cancel_baton, + max_mem, batch, cancel_func, cancel_baton, scratch_pool)); SVN_ERR(svn_io_copy_perms(shard_path, pack_file_dir, scratch_pool)); @@ -2115,11 +2049,10 @@ pack_rev_shard(svn_fs_t *fs, return SVN_NO_ERROR; } -/* In the file system at FS_PATH, pack the SHARD in REVS_DIR and - * REVPROPS_DIR containing exactly MAX_FILES_PER_DIR revisions, using - * SCRATCH_POOL temporary for allocations. REVPROPS_DIR will be NULL if - * revprop packing is not supported. COMPRESSION_LEVEL and MAX_PACK_SIZE - * will be ignored in that case. +/* In the file system at FS_PATH, pack the SHARD in DIR containing exactly + * MAX_FILES_PER_DIR revisions, using SCRATCH_POOL temporary for allocations. + * COMPRESSION_LEVEL and MAX_PACK_SIZE will be ignored in that case. + * An attempt will be made to keep memory usage below MAX_MEM. * * CANCEL_FUNC and CANCEL_BATON are what you think they are; similarly * NOTIFY_FUNC and NOTIFY_BATON. @@ -2128,13 +2061,13 @@ pack_rev_shard(svn_fs_t *fs, * remove the pack file and start again. */ static svn_error_t * -pack_shard(const char *revs_dir, - const char *revsprops_dir, +pack_shard(const char *dir, svn_fs_t *fs, apr_int64_t shard, int max_files_per_dir, apr_off_t max_pack_size, int compression_level, + apr_size_t max_mem, svn_fs_pack_notify_t notify_func, void *notify_baton, svn_cancel_func_t cancel_func, @@ -2142,49 +2075,42 @@ pack_shard(const char *revs_dir, apr_pool_t *scratch_pool) { svn_fs_x__data_t *ffd = fs->fsap_data; - const char *rev_shard_path, *rev_pack_file_dir; - const char *revprops_shard_path, *revprops_pack_file_dir; + const char *shard_path, *pack_file_dir; + svn_fs_x__batch_fsync_t *batch; /* Notify caller we're starting to pack this shard. */ if (notify_func) SVN_ERR(notify_func(notify_baton, shard, svn_fs_pack_notify_start, scratch_pool)); + /* Perform all fsyncs through this instance. */ + SVN_ERR(svn_fs_x__batch_fsync_create(&batch, ffd->flush_to_disk, + scratch_pool)); + /* Some useful paths. */ - rev_pack_file_dir = svn_dirent_join(revs_dir, + pack_file_dir = svn_dirent_join(dir, apr_psprintf(scratch_pool, "%" APR_INT64_T_FMT PATH_EXT_PACKED_SHARD, shard), scratch_pool); - rev_shard_path = svn_dirent_join(revs_dir, + shard_path = svn_dirent_join(dir, apr_psprintf(scratch_pool, "%" APR_INT64_T_FMT, shard), scratch_pool); /* pack the revision content */ - SVN_ERR(pack_rev_shard(fs, rev_pack_file_dir, rev_shard_path, - shard, max_files_per_dir, DEFAULT_MAX_MEM, + SVN_ERR(pack_rev_shard(fs, pack_file_dir, shard_path, + shard, max_files_per_dir, max_mem, batch, cancel_func, cancel_baton, scratch_pool)); - /* if enabled, pack the revprops in an equivalent way */ - if (revsprops_dir) - { - revprops_pack_file_dir = svn_dirent_join(revsprops_dir, - apr_psprintf(scratch_pool, - "%" APR_INT64_T_FMT PATH_EXT_PACKED_SHARD, - shard), - scratch_pool); - revprops_shard_path = svn_dirent_join(revsprops_dir, - apr_psprintf(scratch_pool, "%" APR_INT64_T_FMT, shard), - scratch_pool); - - SVN_ERR(svn_fs_x__pack_revprops_shard(revprops_pack_file_dir, - revprops_shard_path, - shard, max_files_per_dir, - (int)(0.9 * max_pack_size), - compression_level, - cancel_func, cancel_baton, - scratch_pool)); - } + /* pack the revprops in an equivalent way */ + SVN_ERR(svn_fs_x__pack_revprops_shard(fs, + pack_file_dir, + shard_path, + shard, max_files_per_dir, + (int)(0.9 * max_pack_size), + compression_level, batch, + cancel_func, cancel_baton, + scratch_pool)); /* Update the min-unpacked-rev file to reflect our newly packed shard. */ SVN_ERR(svn_fs_x__write_min_unpacked_rev(fs, @@ -2192,35 +2118,12 @@ pack_shard(const char *revs_dir, scratch_pool)); ffd->min_unpacked_rev = (svn_revnum_t)((shard + 1) * max_files_per_dir); - /* Finally, remove the existing shard directories. - * For revprops, clean up older obsolete shards as well as they might - * have been left over from an interrupted FS upgrade. */ - SVN_ERR(svn_io_remove_dir2(rev_shard_path, TRUE, + /* Ensure that packed file is written to disk.*/ + SVN_ERR(svn_fs_x__batch_fsync_run(batch, scratch_pool)); + + /* Finally, remove the existing shard directories. */ + SVN_ERR(svn_io_remove_dir2(shard_path, TRUE, cancel_func, cancel_baton, scratch_pool)); - if (revsprops_dir) - { - svn_node_kind_t kind = svn_node_dir; - apr_int64_t to_cleanup = shard; - do - { - SVN_ERR(svn_fs_x__delete_revprops_shard(revprops_shard_path, - to_cleanup, - max_files_per_dir, - cancel_func, cancel_baton, - scratch_pool)); - - /* If the previous shard exists, clean it up as well. - Don't try to clean up shard 0 as it we can't tell quickly - whether it actually needs cleaning up. */ - revprops_shard_path = svn_dirent_join(revsprops_dir, - apr_psprintf(scratch_pool, - "%" APR_INT64_T_FMT, - --to_cleanup), - scratch_pool); - SVN_ERR(svn_io_check_path(revprops_shard_path, &kind, scratch_pool)); - } - while (kind == svn_node_dir && to_cleanup > 0); - } /* Notify caller we're starting to pack this shard. */ if (notify_func) @@ -2230,9 +2133,38 @@ pack_shard(const char *revs_dir, return SVN_NO_ERROR; } +/* Read the youngest rev and the first non-packed rev info for FS from disk. + Set *FULLY_PACKED when there is no completed unpacked shard. + Use SCRATCH_POOL for temporary allocations. + */ +static svn_error_t * +get_pack_status(svn_boolean_t *fully_packed, + svn_fs_t *fs, + apr_pool_t *scratch_pool) +{ + svn_fs_x__data_t *ffd = fs->fsap_data; + apr_int64_t completed_shards; + svn_revnum_t youngest; + + SVN_ERR(svn_fs_x__read_min_unpacked_rev(&ffd->min_unpacked_rev, fs, + scratch_pool)); + + SVN_ERR(svn_fs_x__youngest_rev(&youngest, fs, scratch_pool)); + completed_shards = (youngest + 1) / ffd->max_files_per_dir; + + /* See if we've already completed all possible shards thus far. */ + if (ffd->min_unpacked_rev == (completed_shards * ffd->max_files_per_dir)) + *fully_packed = TRUE; + else + *fully_packed = FALSE; + + return SVN_NO_ERROR; +} + typedef struct pack_baton_t { svn_fs_t *fs; + apr_size_t max_mem; svn_fs_pack_notify_t notify_func; void *notify_baton; svn_cancel_func_t cancel_func; @@ -2262,25 +2194,25 @@ pack_body(void *baton, svn_fs_x__data_t *ffd = pb->fs->fsap_data; apr_int64_t completed_shards; apr_int64_t i; - svn_revnum_t youngest; apr_pool_t *iterpool; - const char *rev_data_path; - const char *revprops_data_path = NULL; - - /* If we aren't using sharding, we can't do any packing, so quit. */ - SVN_ERR(svn_fs_x__read_min_unpacked_rev(&ffd->min_unpacked_rev, pb->fs, - scratch_pool)); + const char *data_path; + svn_boolean_t fully_packed; - SVN_ERR(svn_fs_x__youngest_rev(&youngest, pb->fs, scratch_pool)); - completed_shards = (youngest + 1) / ffd->max_files_per_dir; + /* Since another process might have already packed the repo, + we need to re-read the pack status. */ + SVN_ERR(get_pack_status(&fully_packed, pb->fs, scratch_pool)); + if (fully_packed) + { + if (pb->notify_func) + (*pb->notify_func)(pb->notify_baton, + ffd->min_unpacked_rev / ffd->max_files_per_dir, + svn_fs_pack_notify_noop, scratch_pool); - /* See if we've already completed all possible shards thus far. */ - if (ffd->min_unpacked_rev == (completed_shards * ffd->max_files_per_dir)) - return SVN_NO_ERROR; + return SVN_NO_ERROR; + } - rev_data_path = svn_dirent_join(pb->fs->path, PATH_REVS_DIR, scratch_pool); - revprops_data_path = svn_dirent_join(pb->fs->path, PATH_REVPROPS_DIR, - scratch_pool); + completed_shards = (ffd->youngest_rev_cache + 1) / ffd->max_files_per_dir; + data_path = svn_dirent_join(pb->fs->path, PATH_REVS_DIR, scratch_pool); iterpool = svn_pool_create(scratch_pool); for (i = ffd->min_unpacked_rev / ffd->max_files_per_dir; @@ -2292,12 +2224,13 @@ pack_body(void *baton, if (pb->cancel_func) SVN_ERR(pb->cancel_func(pb->cancel_baton)); - SVN_ERR(pack_shard(rev_data_path, revprops_data_path, + SVN_ERR(pack_shard(data_path, pb->fs, i, ffd->max_files_per_dir, ffd->revprop_pack_size, ffd->compress_packed_revprops ? SVN__COMPRESSION_ZLIB_DEFAULT : SVN__COMPRESSION_NONE, + pb->max_mem, pb->notify_func, pb->notify_baton, pb->cancel_func, pb->cancel_baton, iterpool)); } @@ -2308,6 +2241,7 @@ pack_body(void *baton, svn_error_t * svn_fs_x__pack(svn_fs_t *fs, + apr_size_t max_mem, svn_fs_pack_notify_t notify_func, void *notify_baton, svn_cancel_func_t cancel_func, @@ -2315,10 +2249,29 @@ svn_fs_x__pack(svn_fs_t *fs, apr_pool_t *scratch_pool) { pack_baton_t pb = { 0 }; + svn_boolean_t fully_packed; + + /* Is there we even anything to do?. */ + SVN_ERR(get_pack_status(&fully_packed, fs, scratch_pool)); + if (fully_packed) + { + svn_fs_x__data_t *ffd = fs->fsap_data; + + if (notify_func) + (*notify_func)(notify_baton, + ffd->min_unpacked_rev / ffd->max_files_per_dir, + svn_fs_pack_notify_noop, scratch_pool); + + return SVN_NO_ERROR; + } + + /* Lock the repo and start the pack process. */ pb.fs = fs; pb.notify_func = notify_func; pb.notify_baton = notify_baton; pb.cancel_func = cancel_func; pb.cancel_baton = cancel_baton; + pb.max_mem = max_mem ? max_mem : DEFAULT_MAX_MEM; + return svn_fs_x__with_pack_lock(fs, pack_body, &pb, scratch_pool); } diff --git a/subversion/libsvn_fs_x/pack.h b/subversion/libsvn_fs_x/pack.h index 55416190f550..9677946c60f4 100644 --- a/subversion/libsvn_fs_x/pack.h +++ b/subversion/libsvn_fs_x/pack.h @@ -20,36 +20,31 @@ * ==================================================================== */ -#ifndef SVN_LIBSVN_FS__PACK_H -#define SVN_LIBSVN_FS__PACK_H +#ifndef SVN_LIBSVN_FS_X_PACK_H +#define SVN_LIBSVN_FS_X_PACK_H #include "fs.h" /* Possibly pack the repository at PATH. This just take full shards, and - combines all the revision files into a single one, with a manifest header. + combines all the revision files into a single one, with a manifest header + when required by the repository format. + + MAX_MEM limits the size of in-memory data structures needed for reordering + items. 0 means use the built-in default. + Use optional CANCEL_FUNC/CANCEL_BATON for cancellation support. Use SCRATCH_POOL for temporary allocations. Existing filesystem references need not change. */ svn_error_t * svn_fs_x__pack(svn_fs_t *fs, + apr_size_t max_mem, svn_fs_pack_notify_t notify_func, void *notify_baton, svn_cancel_func_t cancel_func, void *cancel_baton, apr_pool_t *scratch_pool); -/** - * For the packed revision REV in FS, determine the offset within the - * revision pack file and return it in REV_OFFSET. - * Use SCRATCH_POOL for temporary allocations. - */ -svn_error_t * -svn_fs_x__get_packed_offset(apr_off_t *rev_offset, - svn_fs_t *fs, - svn_revnum_t rev, - apr_pool_t *scratch_pool); - /* Return the svn_dir_entry_t* objects of DIRECTORY in an APR array * allocated in RESULT_POOL with entries added in storage (on-disk) order. * FS' format will be used to pick the optimal ordering strategy. Use diff --git a/subversion/libsvn_fs_x/recovery.c b/subversion/libsvn_fs_x/recovery.c index 984b74023130..f49f6a645148 100644 --- a/subversion/libsvn_fs_x/recovery.c +++ b/subversion/libsvn_fs_x/recovery.c @@ -22,6 +22,7 @@ #include "recovery.h" +#include "svn_dirent_uri.h" #include "svn_hash.h" #include "svn_pools.h" #include "private/svn_string_private.h" @@ -38,6 +39,22 @@ #include "svn_private_config.h" +/* Set *EXISTS to TRUE, if the revision / pack file for REV exists in FS. + Use SCRATCH_POOL for temporary allocations. */ +static svn_error_t * +revision_file_exists(svn_boolean_t *exists, + svn_fs_t *fs, + svn_revnum_t rev, + apr_pool_t *scratch_pool) +{ + svn_node_kind_t kind; + const char *path = svn_fs_x__path_rev_absolute(fs, rev, scratch_pool); + SVN_ERR(svn_io_check_path(path, &kind, scratch_pool)); + + *exists = kind == svn_node_file; + return SVN_NO_ERROR; +} + /* Part of the recovery procedure. Return the largest revision *REV in filesystem FS. Use SCRATCH_POOL for temporary allocation. */ static svn_error_t * @@ -56,19 +73,12 @@ recover_get_largest_revision(svn_fs_t *fs, /* Keep doubling right, until we find a revision that doesn't exist. */ while (1) { - svn_error_t *err; - svn_fs_x__revision_file_t *file; + svn_boolean_t exists; svn_pool_clear(iterpool); - err = svn_fs_x__open_pack_or_rev_file(&file, fs, right, iterpool, - iterpool); - if (err && err->apr_err == SVN_ERR_FS_NO_SUCH_REVISION) - { - svn_error_clear(err); - break; - } - else - SVN_ERR(err); + SVN_ERR(revision_file_exists(&exists, fs, right, iterpool)); + if (!exists) + break; right <<= 1; } @@ -80,22 +90,14 @@ recover_get_largest_revision(svn_fs_t *fs, while (left + 1 < right) { svn_revnum_t probe = left + ((right - left) / 2); - svn_error_t *err; - svn_fs_x__revision_file_t *file; + svn_boolean_t exists; svn_pool_clear(iterpool); - err = svn_fs_x__open_pack_or_rev_file(&file, fs, probe, iterpool, - iterpool); - if (err && err->apr_err == SVN_ERR_FS_NO_SUCH_REVISION) - { - svn_error_clear(err); - right = probe; - } + SVN_ERR(revision_file_exists(&exists, fs, probe, iterpool)); + if (exists) + left = probe; else - { - SVN_ERR(err); - left = probe; - } + right = probe; } svn_pool_destroy(iterpool); @@ -105,6 +107,86 @@ recover_get_largest_revision(svn_fs_t *fs, return SVN_NO_ERROR; } +/* Delete all files and sub-directories (recursively) of DIR_PATH but + leave DIR_PATH itself in place. Use SCRATCH_POOL for temporaries. */ +static svn_error_t * +clear_directory(const char *dir_path, + apr_pool_t *scratch_pool) +{ + apr_hash_t *dirents; + apr_hash_index_t *hi; + apr_pool_t *iterpool = svn_pool_create(scratch_pool); + + SVN_ERR(svn_io_get_dirents3(&dirents, dir_path, TRUE, scratch_pool, + scratch_pool)); + + for (hi = apr_hash_first(scratch_pool, dirents); + hi; + hi = apr_hash_next(hi)) + { + const char *path; + const char *name; + svn_dirent_t *dirent; + + svn_pool_clear(iterpool); + apr_hash_this(hi, (const void **)&name, NULL, (void **)&dirent); + + path = svn_dirent_join(dir_path, name, iterpool); + if (dirent->kind == svn_node_dir) + SVN_ERR(svn_io_remove_dir2(path, TRUE, NULL, NULL, iterpool)); + else + SVN_ERR(svn_io_remove_file2(path, TRUE, iterpool)); + } + + svn_pool_destroy(iterpool); + + return SVN_NO_ERROR; +} + +/* Delete all uncommitted transaction data from FS. + Use SCRATCH_POOL for temporaries. */ +static svn_error_t * +discard_transactions(svn_fs_t *fs, + apr_pool_t *scratch_pool) +{ + svn_fs_x__data_t *ffd = fs->fsap_data; + svn_fs_x__shared_data_t *ffsd = ffd->shared; + + /* In case this FS has been opened more than once in this process, + we should purge their shared transaction data as well. We do the + same as abort_txn would, except that we don't expect all txn files + to be complete on disk. */ + while (ffsd->txns) + { + svn_fs_x__shared_txn_data_t *txn = ffsd->txns; + ffsd->txns = txn->next; + + svn_pool_destroy(txn->pool); + } + + /* Remove anything from the transaction folders. */ + SVN_ERR(clear_directory(svn_fs_x__path_txns_dir(fs, scratch_pool), + scratch_pool)); + SVN_ERR(clear_directory(svn_fs_x__path_txn_proto_revs(fs, scratch_pool), + scratch_pool)); + + return SVN_NO_ERROR; +} + +/* Reset txn-current in FS. Use SCRATCH_POOL for temporaries. */ +static svn_error_t * +reset_txn_number(svn_fs_t *fs, + apr_pool_t *scratch_pool) +{ + const char *initial_txn = "0\n"; + SVN_ERR(svn_io_write_atomic2(svn_fs_x__path_txn_current(fs, scratch_pool), + initial_txn, strlen(initial_txn), + svn_fs_x__path_uuid(fs, scratch_pool), + FALSE, scratch_pool)); + + return SVN_NO_ERROR; +} + /* Baton used for recover_body below. */ typedef struct recover_baton_t { svn_fs_t *fs; @@ -133,7 +215,13 @@ recover_body(void *baton, /* The admin may have created a plain copy of this repo before attempting to recover it (hotcopy may or may not work with corrupted repos). Bump the instance ID. */ - SVN_ERR(svn_fs_x__set_uuid(fs, fs->uuid, NULL, scratch_pool)); + SVN_ERR(svn_fs_x__set_uuid(fs, fs->uuid, NULL, TRUE, scratch_pool)); + + /* Because transactions are not resilient against system crashes, + any existing transaction is suspect (and would probably not be + reopened anyway). Get rid of those. */ + SVN_ERR(discard_transactions(fs, scratch_pool)); + SVN_ERR(reset_txn_number(fs, scratch_pool)); /* We need to know the largest revision in the filesystem. */ SVN_ERR(recover_get_largest_revision(fs, &max_rev, scratch_pool)); diff --git a/subversion/libsvn_fs_x/recovery.h b/subversion/libsvn_fs_x/recovery.h index 4fe0a072adcb..ffb064673e85 100644 --- a/subversion/libsvn_fs_x/recovery.h +++ b/subversion/libsvn_fs_x/recovery.h @@ -20,8 +20,8 @@ * ==================================================================== */ -#ifndef SVN_LIBSVN_FS__RECOVERY_H -#define SVN_LIBSVN_FS__RECOVERY_H +#ifndef SVN_LIBSVN_FS_X_RECOVERY_H +#define SVN_LIBSVN_FS_X_RECOVERY_H #include "fs.h" diff --git a/subversion/libsvn_fs_x/rep-cache-db.h b/subversion/libsvn_fs_x/rep-cache-db.h index 918955f51d9f..c212760651d5 100644 --- a/subversion/libsvn_fs_x/rep-cache-db.h +++ b/subversion/libsvn_fs_x/rep-cache-db.h @@ -1,4 +1,4 @@ -/* This file is automatically generated from rep-cache-db.sql and .dist_sandbox/subversion-1.9.7/subversion/libsvn_fs_x/token-map.h. +/* This file is automatically generated from rep-cache-db.sql and subversion/libsvn_fs_x/token-map.h. * Do not edit this file -- edit the source and rerun gen-make.py */ #define STMT_CREATE_SCHEMA 0 diff --git a/subversion/libsvn_fs_x/rep-cache.c b/subversion/libsvn_fs_x/rep-cache.c index 85e62a46f376..639762182582 100644 --- a/subversion/libsvn_fs_x/rep-cache.c +++ b/subversion/libsvn_fs_x/rep-cache.c @@ -100,12 +100,16 @@ open_rep_cache(void *baton, 0, NULL, 0, fs->pool, scratch_pool)); - SVN_ERR(svn_sqlite__read_schema_version(&version, sdb, scratch_pool)); + SVN_SQLITE__ERR_CLOSE(svn_sqlite__read_schema_version(&version, sdb, + scratch_pool), + sdb); if (version < REP_CACHE_SCHEMA_FORMAT) { /* Must be 0 -- an uninitialized (no schema) database. Create the schema. Results in schema version of 1. */ - SVN_ERR(svn_sqlite__exec_statements(sdb, STMT_CREATE_SCHEMA)); + SVN_SQLITE__ERR_CLOSE(svn_sqlite__exec_statements(sdb, + STMT_CREATE_SCHEMA), + sdb); } /* This is used as a flag that the database is available so don't @@ -122,7 +126,26 @@ svn_fs_x__open_rep_cache(svn_fs_t *fs, svn_fs_x__data_t *ffd = fs->fsap_data; svn_error_t *err = svn_atomic__init_once(&ffd->rep_cache_db_opened, open_rep_cache, fs, scratch_pool); - return svn_error_quick_wrap(err, _("Couldn't open rep-cache database")); + return svn_error_quick_wrapf(err, + _("Couldn't open rep-cache database '%s'"), + svn_dirent_local_style( + path_rep_cache_db(fs->path, scratch_pool), + scratch_pool)); +} + +svn_error_t * +svn_fs_x__close_rep_cache(svn_fs_t *fs) +{ + svn_fs_x__data_t *ffd = fs->fsap_data; + + if (ffd->rep_cache_db) + { + SVN_ERR(svn_sqlite__close(ffd->rep_cache_db)); + ffd->rep_cache_db = NULL; + ffd->rep_cache_db_opened = 0; + } + + return SVN_NO_ERROR; } svn_error_t * @@ -236,7 +259,7 @@ svn_fs_x__walk_rep_reference(svn_fs_t *fs, If you extend this function, check the callsite to see if you have to make it not-ignore additional error codes. */ svn_error_t * -svn_fs_x__get_rep_reference(svn_fs_x__representation_t **rep, +svn_fs_x__get_rep_reference(svn_fs_x__representation_t **rep_p, svn_fs_t *fs, svn_checksum_t *checksum, apr_pool_t *result_pool, @@ -245,6 +268,7 @@ svn_fs_x__get_rep_reference(svn_fs_x__representation_t **rep, svn_fs_x__data_t *ffd = fs->fsap_data; svn_sqlite__stmt_t *stmt; svn_boolean_t have_row; + svn_fs_x__representation_t *rep; SVN_ERR_ASSERT(ffd->rep_sharing_allowed); if (! ffd->rep_cache_db) @@ -263,24 +287,23 @@ svn_fs_x__get_rep_reference(svn_fs_x__representation_t **rep, SVN_ERR(svn_sqlite__step(&have_row, stmt)); if (have_row) { - *rep = apr_pcalloc(result_pool, sizeof(**rep)); - memcpy((*rep)->sha1_digest, checksum->digest, - sizeof((*rep)->sha1_digest)); - (*rep)->has_sha1 = TRUE; - (*rep)->id.change_set = svn_sqlite__column_revnum(stmt, 0); - (*rep)->id.number = svn_sqlite__column_int64(stmt, 1); - (*rep)->size = svn_sqlite__column_int64(stmt, 2); - (*rep)->expanded_size = svn_sqlite__column_int64(stmt, 3); + rep = apr_pcalloc(result_pool, sizeof(*rep)); + memcpy(rep->sha1_digest, checksum->digest, sizeof(rep->sha1_digest)); + rep->has_sha1 = TRUE; + rep->id.change_set = svn_sqlite__column_revnum(stmt, 0); + rep->id.number = svn_sqlite__column_int64(stmt, 1); + rep->size = svn_sqlite__column_int64(stmt, 2); + rep->expanded_size = svn_sqlite__column_int64(stmt, 3); } else - *rep = NULL; + rep = NULL; SVN_ERR(svn_sqlite__reset(stmt)); - if (*rep) + if (rep) { /* Check that REP refers to a revision that exists in FS. */ - svn_revnum_t revision = svn_fs_x__get_revnum((*rep)->id.change_set); + svn_revnum_t revision = svn_fs_x__get_revnum(rep->id.change_set); svn_error_t *err = svn_fs_x__ensure_revision_exists(revision, fs, scratch_pool); if (err) @@ -289,6 +312,7 @@ svn_fs_x__get_rep_reference(svn_fs_x__representation_t **rep, svn_checksum_to_cstring_display(checksum, scratch_pool)); } + *rep_p = rep; return SVN_NO_ERROR; } diff --git a/subversion/libsvn_fs_x/rep-cache.h b/subversion/libsvn_fs_x/rep-cache.h index 1fe26da9b810..5c355cbbe03e 100644 --- a/subversion/libsvn_fs_x/rep-cache.h +++ b/subversion/libsvn_fs_x/rep-cache.h @@ -40,6 +40,10 @@ svn_error_t * svn_fs_x__open_rep_cache(svn_fs_t *fs, apr_pool_t *scratch_pool); +/* Close the rep cache database associated with FS. */ +svn_error_t * +svn_fs_x__close_rep_cache(svn_fs_t *fs); + /* Set *EXISTS to TRUE iff the rep-cache DB file exists. */ svn_error_t * svn_fs_x__exists_rep_cache(svn_boolean_t *exists, @@ -61,11 +65,12 @@ svn_fs_x__walk_rep_reference(svn_fs_t *fs, apr_pool_t *scratch_pool); /* Return the representation REP in FS which has fulltext CHECKSUM. - REP is allocated in RESULT_POOL. If the rep cache database has not been - opened, just set *REP to NULL. Returns SVN_ERR_FS_CORRUPT if a reference - beyond HEAD is detected. Uses SCRATCH_POOL for temporary allocations. */ + *REP_P is allocated in RESULT_POOL. If the rep cache database has not + been opened, just set *REP_P to NULL. Returns SVN_ERR_FS_CORRUPT if + a reference beyond HEAD is detected. Uses SCRATCH_POOL for temporary + allocations.*/ svn_error_t * -svn_fs_x__get_rep_reference(svn_fs_x__representation_t **rep, +svn_fs_x__get_rep_reference(svn_fs_x__representation_t **rep_p, svn_fs_t *fs, svn_checksum_t *checksum, apr_pool_t *result_pool, diff --git a/subversion/libsvn_fs_x/reps.c b/subversion/libsvn_fs_x/reps.c index 85a5269a7fc3..0bfc2412237c 100644 --- a/subversion/libsvn_fs_x/reps.c +++ b/subversion/libsvn_fs_x/reps.c @@ -417,8 +417,8 @@ svn_fs_x__reps_add_base(svn_fs_x__reps_builder_t *builder, apr_size_t idx; SVN_ERR(svn_fs_x__get_contents(&stream, builder->fs, rep, FALSE, scratch_pool)); - SVN_ERR(svn_string_from_stream(&contents, stream, scratch_pool, - scratch_pool)); + SVN_ERR(svn_string_from_stream2(&contents, stream, SVN__STREAM_CHUNK_SIZE, + scratch_pool)); SVN_ERR(svn_fs_x__reps_add(&idx, builder, contents)); base.revision = svn_fs_x__get_revnum(rep->id.change_set); @@ -641,16 +641,17 @@ svn_fs_x__reps_get(svn_fs_x__rep_extractor_t **extractor, svn_fs_t *fs, const svn_fs_x__reps_t *container, apr_size_t idx, - apr_pool_t *pool) + apr_pool_t *result_pool) { apr_uint32_t first = container->first_instructions[idx]; apr_uint32_t last = container->first_instructions[idx + 1]; /* create the extractor object */ - svn_fs_x__rep_extractor_t *result = apr_pcalloc(pool, sizeof(*result)); + svn_fs_x__rep_extractor_t *result = apr_pcalloc(result_pool, + sizeof(*result)); result->fs = fs; - result->result = svn_stringbuf_create_empty(pool); - result->pool = pool; + result->result = svn_stringbuf_create_empty(result_pool); + result->pool = result_pool; /* fill all the bits of the result that we can, i.e. all but bits coming * from base representations */ @@ -900,7 +901,7 @@ svn_error_t * svn_fs_x__deserialize_reps_container(void **out, void *data, apr_size_t data_len, - apr_pool_t *pool) + apr_pool_t *result_pool) { svn_fs_x__reps_t *reps = (svn_fs_x__reps_t *)data; diff --git a/subversion/libsvn_fs_x/reps.h b/subversion/libsvn_fs_x/reps.h index 720bfbfb4290..8edd21361d44 100644 --- a/subversion/libsvn_fs_x/reps.h +++ b/subversion/libsvn_fs_x/reps.h @@ -20,8 +20,8 @@ * ==================================================================== */ -#ifndef SVN_LIBSVN_FS__REPS_H -#define SVN_LIBSVN_FS__REPS_H +#ifndef SVN_LIBSVN_FS_X_REPS_H +#define SVN_LIBSVN_FS_X_REPS_H #include "svn_io.h" #include "fs.h" @@ -112,14 +112,14 @@ svn_fs_x__reps_estimate_size(const svn_fs_x__reps_builder_t *builder); /* Read from representation containers. */ /* For fulltext IDX in CONTAINER in filesystem FS, create an extract object - * allocated in POOL and return it in *EXTRACTOR. + * allocated in RESULT_POOL and return it in *EXTRACTOR. */ svn_error_t * svn_fs_x__reps_get(svn_fs_x__rep_extractor_t **extractor, svn_fs_t *fs, const svn_fs_x__reps_t *container, apr_size_t idx, - apr_pool_t *pool); + apr_pool_t *result_pool); /* Let the EXTRACTOR object fetch all parts of the desired fulltext and * return the latter in *CONTENTS. If SIZE is not 0, return SIZE bytes @@ -172,7 +172,7 @@ svn_error_t * svn_fs_x__deserialize_reps_container(void **out, void *data, apr_size_t data_len, - apr_pool_t *pool); + apr_pool_t *result_pool); /* Implements svn_cache__partial_getter_func_t for svn_fs_x__reps_t, * setting *OUT to an svn_fs_x__rep_extractor_t object defined by the diff --git a/subversion/libsvn_fs_x/rev_file.c b/subversion/libsvn_fs_x/rev_file.c index 445d45b55b43..2a362337a93c 100644 --- a/subversion/libsvn_fs_x/rev_file.c +++ b/subversion/libsvn_fs_x/rev_file.c @@ -20,6 +20,8 @@ * ==================================================================== */ +#include "svn_pools.h" + #include "rev_file.h" #include "fs_x.h" #include "index.h" @@ -31,6 +33,46 @@ #include "private/svn_io_private.h" #include "svn_private_config.h" +struct svn_fs_x__revision_file_t +{ + /* the filesystem that this revision file belongs to */ + svn_fs_t *fs; + + /* Meta-data to FILE. */ + svn_fs_x__rev_file_info_t file_info; + + /* rev / pack file */ + apr_file_t *file; + + /* stream based on FILE and not NULL exactly when FILE is not NULL */ + svn_stream_t *stream; + + /* the opened P2L index stream or NULL. Always NULL for txns. */ + svn_fs_x__packed_number_stream_t *p2l_stream; + + /* the opened L2P index stream or NULL. Always NULL for txns. */ + svn_fs_x__packed_number_stream_t *l2p_stream; + + /* Copied from FS->FFD->BLOCK_SIZE upon creation. It allows us to + * use aligned seek() without having the FS handy. */ + apr_off_t block_size; + + /* Info on the L2P index within FILE. + * Elements are -1 / NULL until svn_fs_x__auto_read_footer gets called. */ + svn_fs_x__index_info_t l2p_info; + + /* Info on the P2L index within FILE. + * Elements are -1 / NULL until svn_fs_x__auto_read_footer gets called. */ + svn_fs_x__index_info_t p2l_info; + + /* Pool used for all sub-structure allocations (file, streams etc.). + A sub-pool of OWNER. NULL until the lazily initilized. */ + apr_pool_t *pool; + + /* Pool that this structure got allocated in. */ + apr_pool_t *owner; +}; + /* Return a new revision file instance, allocated in RESULT_POOL, for * filesystem FS. Set its pool member to the provided RESULT_POOL. */ static svn_fs_x__revision_file_t * @@ -38,22 +80,24 @@ create_revision_file(svn_fs_t *fs, apr_pool_t *result_pool) { svn_fs_x__data_t *ffd = fs->fsap_data; - svn_fs_x__revision_file_t *file = apr_palloc(result_pool, sizeof(*file)); - - file->is_packed = FALSE; - file->start_revision = SVN_INVALID_REVNUM; + svn_fs_x__revision_file_t *file = apr_palloc(result_pool, sizeof(*file)); + file->fs = fs; + file->file_info.is_packed = FALSE; + file->file_info.start_revision = SVN_INVALID_REVNUM; file->file = NULL; file->stream = NULL; file->p2l_stream = NULL; file->l2p_stream = NULL; file->block_size = ffd->block_size; - file->l2p_offset = -1; - file->l2p_checksum = NULL; - file->p2l_offset = -1; - file->p2l_checksum = NULL; - file->footer_offset = -1; - file->pool = result_pool; + file->l2p_info.start = -1; + file->l2p_info.end = -1; + file->l2p_info.checksum = NULL; + file->p2l_info.start = -1; + file->p2l_info.end = -1; + file->p2l_info.checksum = NULL; + file->pool = NULL; + file->owner = result_pool; return file; } @@ -68,8 +112,8 @@ init_revision_file(svn_fs_t *fs, { svn_fs_x__revision_file_t *file = create_revision_file(fs, result_pool); - file->is_packed = svn_fs_x__is_packed_rev(fs, revision); - file->start_revision = svn_fs_x__packed_base_rev(fs, revision); + file->file_info.is_packed = svn_fs_x__is_packed_rev(fs, revision); + file->file_info.start_revision = svn_fs_x__packed_base_rev(fs, revision); return file; } @@ -137,20 +181,31 @@ auto_make_writable(const char *path, return SVN_NO_ERROR; } -/* Core implementation of svn_fs_fs__open_pack_or_rev_file working on an +/* Return the pool to be used for allocations with FILE. + Lazily created that pool upon the first call. */ +static apr_pool_t * +get_file_pool(svn_fs_x__revision_file_t *file) +{ + if (file->pool == NULL) + file->pool = svn_pool_create(file->owner); + + return file->pool; +} + +/* Core implementation of svn_fs_x__open_pack_or_rev_file working on an * existing, initialized FILE structure. If WRITABLE is TRUE, give write * access to the file - temporarily resetting the r/o state if necessary. */ static svn_error_t * open_pack_or_rev_file(svn_fs_x__revision_file_t *file, - svn_fs_t *fs, - svn_revnum_t rev, svn_boolean_t writable, - apr_pool_t *result_pool, apr_pool_t *scratch_pool) { svn_error_t *err; svn_boolean_t retry = FALSE; + svn_fs_t *fs = file->fs; + svn_revnum_t rev = file->file_info.start_revision; + apr_pool_t *file_pool = get_file_pool(file); do { @@ -161,19 +216,19 @@ open_pack_or_rev_file(svn_fs_x__revision_file_t *file, : APR_READ | APR_BUFFERED; /* We may have to *temporarily* enable write access. */ - err = writable ? auto_make_writable(path, result_pool, scratch_pool) + err = writable ? auto_make_writable(path, file_pool, scratch_pool) : SVN_NO_ERROR; /* open the revision file in buffered r/o or r/w mode */ if (!err) err = svn_io_file_open(&apr_file, path, flags, APR_OS_DEFAULT, - result_pool); + file_pool); if (!err) { file->file = apr_file; file->stream = svn_stream_from_aprfile2(apr_file, TRUE, - result_pool); + file_pool); return SVN_NO_ERROR; } @@ -191,7 +246,7 @@ open_pack_or_rev_file(svn_fs_x__revision_file_t *file, /* We failed for the first time. Refresh cache & retry. */ SVN_ERR(svn_fs_x__update_min_unpacked_rev(fs, scratch_pool)); - file->start_revision = svn_fs_x__packed_base_rev(fs, rev); + file->file_info.start_revision = svn_fs_x__packed_base_rev(fs, rev); retry = TRUE; } @@ -206,39 +261,52 @@ open_pack_or_rev_file(svn_fs_x__revision_file_t *file, } svn_error_t * -svn_fs_x__open_pack_or_rev_file(svn_fs_x__revision_file_t **file, - svn_fs_t *fs, - svn_revnum_t rev, - apr_pool_t *result_pool, - apr_pool_t *scratch_pool) +svn_fs_x__rev_file_init(svn_fs_x__revision_file_t **file, + svn_fs_t *fs, + svn_revnum_t rev, + apr_pool_t *result_pool) { *file = init_revision_file(fs, rev, result_pool); - return svn_error_trace(open_pack_or_rev_file(*file, fs, rev, FALSE, - result_pool, scratch_pool)); + return SVN_NO_ERROR; } svn_error_t * -svn_fs_x__open_pack_or_rev_file_writable(svn_fs_x__revision_file_t** file, - svn_fs_t* fs, - svn_revnum_t rev, - apr_pool_t* result_pool, - apr_pool_t *scratch_pool) +svn_fs_x__rev_file_open_writable(svn_fs_x__revision_file_t** file, + svn_fs_t* fs, + svn_revnum_t rev, + apr_pool_t* result_pool, + apr_pool_t *scratch_pool) { *file = init_revision_file(fs, rev, result_pool); - return svn_error_trace(open_pack_or_rev_file(*file, fs, rev, TRUE, - result_pool, scratch_pool)); + return svn_error_trace(open_pack_or_rev_file(*file, TRUE, scratch_pool)); } -svn_error_t * -svn_fs_x__auto_read_footer(svn_fs_x__revision_file_t *file) +/* If the revision file in FILE has not been opened, yet, do it now. */ +static svn_error_t * +auto_open(svn_fs_x__revision_file_t *file) { - if (file->l2p_offset == -1) + if (file->file == NULL) + SVN_ERR(open_pack_or_rev_file(file, FALSE, get_file_pool(file))); + + return SVN_NO_ERROR; +} + +/* If the footer data in FILE has not been read, yet, do so now. + * Index locations will only be read upon request as we assume they get + * cached and the FILE is usually used for REP data access only. + * Hence, the separate step. + */ +static svn_error_t * +auto_read_footer(svn_fs_x__revision_file_t *file) +{ + if (file->l2p_info.start == -1) { apr_off_t filesize = 0; unsigned char footer_length; svn_stringbuf_t *footer; /* Determine file size. */ + SVN_ERR(auto_open(file)); SVN_ERR(svn_io_file_seek(file->file, APR_END, &filesize, file->pool)); /* Read last byte (containing the length of the footer). */ @@ -258,22 +326,25 @@ svn_fs_x__auto_read_footer(svn_fs_x__revision_file_t *file) footer->data[footer->len] = '\0'; /* Extract index locations. */ - SVN_ERR(svn_fs_x__parse_footer(&file->l2p_offset, &file->l2p_checksum, - &file->p2l_offset, &file->p2l_checksum, - footer, file->start_revision, - file->pool)); - file->footer_offset = filesize - footer_length - 1; + SVN_ERR(svn_fs_x__parse_footer(&file->l2p_info.start, + &file->l2p_info.checksum, + &file->p2l_info.start, + &file->p2l_info.checksum, + footer, file->file_info.start_revision, + filesize - footer_length - 1, file->pool)); + file->l2p_info.end = file->p2l_info.start; + file->p2l_info.end = filesize - footer_length - 1; } return SVN_NO_ERROR; } svn_error_t * -svn_fs_x__open_proto_rev_file(svn_fs_x__revision_file_t **file, - svn_fs_t *fs, - svn_fs_x__txn_id_t txn_id, - apr_pool_t* result_pool, - apr_pool_t *scratch_pool) +svn_fs_x__rev_file_open_proto_rev(svn_fs_x__revision_file_t **file, + svn_fs_t *fs, + svn_fs_x__txn_id_t txn_id, + apr_pool_t* result_pool, + apr_pool_t *scratch_pool) { apr_file_t *apr_file; SVN_ERR(svn_io_file_open(&apr_file, @@ -282,12 +353,12 @@ svn_fs_x__open_proto_rev_file(svn_fs_x__revision_file_t **file, APR_READ | APR_BUFFERED, APR_OS_DEFAULT, result_pool)); - return svn_error_trace(svn_fs_x__wrap_temp_rev_file(file, fs, apr_file, + return svn_error_trace(svn_fs_x__rev_file_wrap_temp(file, fs, apr_file, result_pool)); } svn_error_t * -svn_fs_x__wrap_temp_rev_file(svn_fs_x__revision_file_t **file, +svn_fs_x__rev_file_wrap_temp(svn_fs_x__revision_file_t **file, svn_fs_t *fs, apr_file_t *temp_file, apr_pool_t *result_pool) @@ -300,17 +371,169 @@ svn_fs_x__wrap_temp_rev_file(svn_fs_x__revision_file_t **file, } svn_error_t * +svn_fs_x__rev_file_info(svn_fs_x__rev_file_info_t *info, + svn_fs_x__revision_file_t *file) +{ + SVN_ERR(auto_open(file)); + + *info = file->file_info; + return SVN_NO_ERROR; +} + +svn_error_t * +svn_fs_x__rev_file_name(const char **filename, + svn_fs_x__revision_file_t *file, + apr_pool_t *result_pool) +{ + SVN_ERR(auto_open(file)); + + return svn_error_trace(svn_io_file_name_get(filename, file->file, + result_pool)); +} + +svn_error_t * +svn_fs_x__rev_file_stream(svn_stream_t **stream, + svn_fs_x__revision_file_t *file) +{ + SVN_ERR(auto_open(file)); + + *stream = file->stream; + return SVN_NO_ERROR; +} + +svn_error_t * +svn_fs_x__rev_file_get(apr_file_t **apr_file, + svn_fs_x__revision_file_t *file) +{ + SVN_ERR(auto_open(file)); + + *apr_file = file->file; + return SVN_NO_ERROR; +} + +svn_error_t * +svn_fs_x__rev_file_l2p_index(svn_fs_x__packed_number_stream_t **stream, + svn_fs_x__revision_file_t *file) +{ + if (file->l2p_stream == NULL) + { + SVN_ERR(auto_read_footer(file)); + SVN_ERR(svn_fs_x__packed_stream_open(&file->l2p_stream, + file->file, + file->l2p_info.start, + file->l2p_info.end, + SVN_FS_X__L2P_STREAM_PREFIX, + (apr_size_t)file->block_size, + file->pool, + file->pool)); + } + + *stream = file->l2p_stream; + return SVN_NO_ERROR; +} + +svn_error_t * +svn_fs_x__rev_file_p2l_index(svn_fs_x__packed_number_stream_t **stream, + svn_fs_x__revision_file_t *file) +{ + if (file->p2l_stream== NULL) + { + SVN_ERR(auto_read_footer(file)); + SVN_ERR(svn_fs_x__packed_stream_open(&file->p2l_stream, + file->file, + file->p2l_info.start, + file->p2l_info.end, + SVN_FS_X__P2L_STREAM_PREFIX, + (apr_size_t)file->block_size, + file->pool, + file->pool)); + } + + *stream = file->p2l_stream; + return SVN_NO_ERROR; +} + +svn_error_t * +svn_fs_x__rev_file_l2p_info(svn_fs_x__index_info_t *info, + svn_fs_x__revision_file_t *file) +{ + SVN_ERR(auto_read_footer(file)); + *info = file->l2p_info; + + return SVN_NO_ERROR; +} + +svn_error_t * +svn_fs_x__rev_file_p2l_info(svn_fs_x__index_info_t *info, + svn_fs_x__revision_file_t *file) +{ + SVN_ERR(auto_read_footer(file)); + *info = file->p2l_info; + + return SVN_NO_ERROR; +} + +svn_error_t * +svn_fs_x__rev_file_data_size(svn_filesize_t *size, + svn_fs_x__revision_file_t *file) +{ + SVN_ERR(auto_read_footer(file)); + *size = file->l2p_info.start; + + return SVN_NO_ERROR; +} + +svn_error_t * +svn_fs_x__rev_file_seek(svn_fs_x__revision_file_t *file, + apr_off_t *buffer_start, + apr_off_t offset) +{ + SVN_ERR(auto_open(file)); + return svn_error_trace(svn_io_file_aligned_seek(file->file, + file->block_size, + buffer_start, offset, + file->pool)); +} + +svn_error_t * +svn_fs_x__rev_file_offset(apr_off_t *offset, + svn_fs_x__revision_file_t *file) +{ + SVN_ERR(auto_open(file)); + return svn_error_trace(svn_io_file_get_offset(offset, file->file, + file->pool)); +} + +svn_error_t * +svn_fs_x__rev_file_read(svn_fs_x__revision_file_t *file, + void *buf, + apr_size_t nbytes) +{ + SVN_ERR(auto_open(file)); + return svn_error_trace(svn_io_file_read_full2(file->file, buf, nbytes, + NULL, NULL, file->pool)); +} + +svn_error_t * svn_fs_x__close_revision_file(svn_fs_x__revision_file_t *file) { + /* Close sub-objects properly */ if (file->stream) SVN_ERR(svn_stream_close(file->stream)); if (file->file) SVN_ERR(svn_io_file_close(file->file, file->pool)); + /* Release the memory. */ + if (file->pool) + svn_pool_clear(file->pool); + + /* Reset pointers to objects previously allocated from FILE->POOL. */ file->file = NULL; file->stream = NULL; file->l2p_stream = NULL; file->p2l_stream = NULL; + /* Cause any index data getters to re-read the footer. */ + file->l2p_info.start = -1; return SVN_NO_ERROR; } diff --git a/subversion/libsvn_fs_x/rev_file.h b/subversion/libsvn_fs_x/rev_file.h index b96d0351883d..8fa791ab7ea8 100644 --- a/subversion/libsvn_fs_x/rev_file.h +++ b/subversion/libsvn_fs_x/rev_file.h @@ -26,11 +26,14 @@ #include "svn_fs.h" #include "id.h" -/* In format 7, index files must be read in sync with the respective - * revision / pack file. I.e. we must use packed index files for packed - * rev files and unpacked ones for non-packed rev files. So, the whole - * point is to open them with matching "is packed" setting in case some - * background pack process was run. +/* In FSX, index data must be read in sync with the respective revision / + * pack file. I.e. we must use packed index files for packed rev files and + * unpacked ones for non-packed rev files. So, the whole point is to open + * them with matching "is packed" setting in case some background pack + * process was run. + * + * Another thing that this allows us is to lazily open the file, i.e. open + * it upon first access. */ /* Opaque index stream type. @@ -38,11 +41,22 @@ typedef struct svn_fs_x__packed_number_stream_t svn_fs_x__packed_number_stream_t; -/* Data file, including indexes data, and associated properties for - * START_REVISION. As the FILE is kept open, background pack operations - * will not cause access to this file to fail. - */ -typedef struct svn_fs_x__revision_file_t +/* Location and content meta data for an index. */ +typedef struct svn_fs_x__index_info_t +{ + /* Offset within the pack / rev file at which the index data starts. */ + apr_off_t start; + + /* First offset behind the index data. */ + apr_off_t end; + + /* MD5 checksum on the whole on-disk representation of the index. */ + svn_checksum_t *checksum; + +} svn_fs_x__index_info_t; + +/* Location and content meta data for a revision / pack file. */ +typedef struct svn_fs_x__rev_file_info_t { /* first (potentially only) revision in the rev / pack file. * SVN_INVALID_REVNUM for txn proto-rev files. */ @@ -51,60 +65,24 @@ typedef struct svn_fs_x__revision_file_t /* the revision was packed when the first file / stream got opened */ svn_boolean_t is_packed; - /* rev / pack file */ - apr_file_t *file; - - /* stream based on FILE and not NULL exactly when FILE is not NULL */ - svn_stream_t *stream; - - /* the opened P2L index stream or NULL. Always NULL for txns. */ - svn_fs_x__packed_number_stream_t *p2l_stream; - - /* the opened L2P index stream or NULL. Always NULL for txns. */ - svn_fs_x__packed_number_stream_t *l2p_stream; - - /* Copied from FS->FFD->BLOCK_SIZE upon creation. It allows us to - * use aligned seek() without having the FS handy. */ - apr_off_t block_size; - - /* Offset within FILE at which the rev data ends and the L2P index - * data starts. Less than P2L_OFFSET. -1 if svn_fs_fs__auto_read_footer - * has not been called, yet. */ - apr_off_t l2p_offset; - - /* MD5 checksum on the whole on-disk representation of the L2P index. - * NULL if svn_fs_fs__auto_read_footer has not been called, yet. */ - svn_checksum_t *l2p_checksum; +} svn_fs_x__rev_file_info_t; - /* Offset within FILE at which the L2P index ends and the P2L index - * data starts. Greater than L2P_OFFSET. -1 if svn_fs_fs__auto_read_footer - * has not been called, yet. */ - apr_off_t p2l_offset; - - /* MD5 checksum on the whole on-disk representation of the P2L index. - * NULL if svn_fs_fs__auto_read_footer has not been called, yet. */ - svn_checksum_t *p2l_checksum; - - /* Offset within FILE at which the P2L index ends and the footer starts. - * Greater than P2L_OFFSET. -1 if svn_fs_fs__auto_read_footer has not - * been called, yet. */ - apr_off_t footer_offset; - - /* pool containing this object */ - apr_pool_t *pool; -} svn_fs_x__revision_file_t; +/* Data file, including indexes data, and associated properties for + * START_REVISION. As the FILE is kept open, background pack operations + * will not cause access to this file to fail. + */ +typedef struct svn_fs_x__revision_file_t svn_fs_x__revision_file_t; -/* Open the correct revision file for REV. If the filesystem FS has - * been packed, *FILE will be set to the packed file; otherwise, set *FILE - * to the revision file for REV. Return SVN_ERR_FS_NO_SUCH_REVISION if the - * file doesn't exist. Allocate *FILE in RESULT_POOL and use SCRATCH_POOL - * for temporaries. */ +/* Initialize the revision / pack file access structure in *FILE for reading + * revision REV from filesystem FS. The file will not be opened until the + * first call to any of the access functions. + * + * Allocate *FILE in RESULT_POOL. */ svn_error_t * -svn_fs_x__open_pack_or_rev_file(svn_fs_x__revision_file_t **file, - svn_fs_t *fs, - svn_revnum_t rev, - apr_pool_t *result_pool, - apr_pool_t *scratch_pool); +svn_fs_x__rev_file_init(svn_fs_x__revision_file_t **file, + svn_fs_t *fs, + svn_revnum_t rev, + apr_pool_t *result_pool); /* Open the correct revision file for REV with read and write access. * If necessary, temporarily reset the file's read-only state. If the @@ -114,39 +92,107 @@ svn_fs_x__open_pack_or_rev_file(svn_fs_x__revision_file_t **file, * Return SVN_ERR_FS_NO_SUCH_REVISION if the file doesn't exist. * Allocate *FILE in RESULT_POOL and use SCRATCH_POOLfor temporaries. */ svn_error_t * -svn_fs_x__open_pack_or_rev_file_writable(svn_fs_x__revision_file_t **file, - svn_fs_t *fs, - svn_revnum_t rev, - apr_pool_t *result_pool, - apr_pool_t *scratch_pool); - -/* If the footer data in FILE has not been read, yet, do so now. - * Index locations will only be read upon request as we assume they get - * cached and the FILE is usually used for REP data access only. - * Hence, the separate step. - */ -svn_error_t * -svn_fs_x__auto_read_footer(svn_fs_x__revision_file_t *file); +svn_fs_x__rev_file_open_writable(svn_fs_x__revision_file_t **file, + svn_fs_t *fs, + svn_revnum_t rev, + apr_pool_t *result_pool, + apr_pool_t *scratch_pool); /* Open the proto-rev file of transaction TXN_ID in FS and return it in *FILE. * Allocate *FILE in RESULT_POOL use and SCRATCH_POOL for temporaries.. */ svn_error_t * -svn_fs_x__open_proto_rev_file(svn_fs_x__revision_file_t **file, - svn_fs_t *fs, - svn_fs_x__txn_id_t txn_id, - apr_pool_t* result_pool, - apr_pool_t *scratch_pool); +svn_fs_x__rev_file_open_proto_rev(svn_fs_x__revision_file_t **file, + svn_fs_t *fs, + svn_fs_x__txn_id_t txn_id, + apr_pool_t* result_pool, + apr_pool_t *scratch_pool); /* Wrap the TEMP_FILE, used in the context of FS, into a revision file * struct, allocated in RESULT_POOL, and return it in *FILE. */ svn_error_t * -svn_fs_x__wrap_temp_rev_file(svn_fs_x__revision_file_t **file, +svn_fs_x__rev_file_wrap_temp(svn_fs_x__revision_file_t **file, svn_fs_t *fs, apr_file_t *temp_file, apr_pool_t *result_pool); -/* Close all files and streams in FILE. +/* Access functions */ + +/* Copy the L2P index info for FILE into *INFO. + */ +svn_error_t * +svn_fs_x__rev_file_info(svn_fs_x__rev_file_info_t *info, + svn_fs_x__revision_file_t *file); + +/* Convenience wrapper around svn_io_file_name_get. */ +svn_error_t * +svn_fs_x__rev_file_name(const char **filename, + svn_fs_x__revision_file_t *file, + apr_pool_t *result_pool); + +/* Set *STREAM to the shared stream object of FILE. + */ +svn_error_t * +svn_fs_x__rev_file_stream(svn_stream_t **stream, + svn_fs_x__revision_file_t *file); + +/* Set *APR_FILE to the shared file object of FILE. + */ +svn_error_t * +svn_fs_x__rev_file_get(apr_file_t **apr_file, + svn_fs_x__revision_file_t *file); + +/* Set *STREAM to the shared L2P data stream of FILE. + */ +svn_error_t * +svn_fs_x__rev_file_l2p_index(svn_fs_x__packed_number_stream_t **stream, + svn_fs_x__revision_file_t *file); + +/* Set *STREAM to the shared P2L data stream of FILE. + */ +svn_error_t * +svn_fs_x__rev_file_p2l_index(svn_fs_x__packed_number_stream_t **stream, + svn_fs_x__revision_file_t *file); + +/* Copy the L2P index info for FILE into *INFO. + */ +svn_error_t * +svn_fs_x__rev_file_l2p_info(svn_fs_x__index_info_t *info, + svn_fs_x__revision_file_t *file); + +/* Copy the P2L index info for FILE into *INFO. + */ +svn_error_t * +svn_fs_x__rev_file_p2l_info(svn_fs_x__index_info_t *info, + svn_fs_x__revision_file_t *file); + +/* Set *SIZE to the length of the revision data in FILE. + */ +svn_error_t * +svn_fs_x__rev_file_data_size(svn_filesize_t *size, + svn_fs_x__revision_file_t *file); + +/* File manipulation. */ + +/* Convenience wrapper around svn_io_file_aligned_seek. */ +svn_error_t * +svn_fs_x__rev_file_seek(svn_fs_x__revision_file_t *file, + apr_off_t *buffer_start, + apr_off_t offset); + +/* Convenience wrapper around svn_fs_x__get_file_offset. */ +svn_error_t * +svn_fs_x__rev_file_offset(apr_off_t *offset, + svn_fs_x__revision_file_t *file); + +/* Convenience wrapper around svn_io_file_read_full2. */ +svn_error_t * +svn_fs_x__rev_file_read(svn_fs_x__revision_file_t *file, + void *buf, + apr_size_t nbytes); + +/* Close all files and streams in FILE. They will be reopened automatically + * by any of the above access functions. */ svn_error_t * svn_fs_x__close_revision_file(svn_fs_x__revision_file_t *file); diff --git a/subversion/libsvn_fs_x/revprops.c b/subversion/libsvn_fs_x/revprops.c index 5bc62ccc1721..f98c04478f24 100644 --- a/subversion/libsvn_fs_x/revprops.c +++ b/subversion/libsvn_fs_x/revprops.c @@ -26,12 +26,16 @@ #include "svn_pools.h" #include "svn_hash.h" #include "svn_dirent_uri.h" +#include "svn_sorts.h" #include "fs_x.h" +#include "low_level.h" #include "revprops.h" #include "util.h" #include "transaction.h" +#include "private/svn_packed_data.h" +#include "private/svn_sorts_private.h" #include "private/svn_subr_private.h" #include "private/svn_string_private.h" #include "../libsvn_fs/fs-loader.h" @@ -48,102 +52,6 @@ giving up. */ #define GENERATION_READ_RETRY_COUNT 100 -/* Maximum size of the generation number file contents (including NUL). */ -#define CHECKSUMMED_NUMBER_BUFFER_LEN \ - (SVN_INT64_BUFFER_SIZE + 3 + APR_MD5_DIGESTSIZE * 2) - - -svn_error_t * -svn_fs_x__upgrade_pack_revprops(svn_fs_t *fs, - svn_fs_upgrade_notify_t notify_func, - void *notify_baton, - svn_cancel_func_t cancel_func, - void *cancel_baton, - apr_pool_t *scratch_pool) -{ - svn_fs_x__data_t *ffd = fs->fsap_data; - const char *revprops_shard_path; - const char *revprops_pack_file_dir; - apr_int64_t shard; - apr_int64_t first_unpacked_shard - = ffd->min_unpacked_rev / ffd->max_files_per_dir; - - apr_pool_t *iterpool = svn_pool_create(scratch_pool); - const char *revsprops_dir = svn_dirent_join(fs->path, PATH_REVPROPS_DIR, - scratch_pool); - int compression_level = ffd->compress_packed_revprops - ? SVN_DELTA_COMPRESSION_LEVEL_DEFAULT - : SVN_DELTA_COMPRESSION_LEVEL_NONE; - - /* first, pack all revprops shards to match the packed revision shards */ - for (shard = 0; shard < first_unpacked_shard; ++shard) - { - svn_pool_clear(iterpool); - - revprops_pack_file_dir = svn_dirent_join(revsprops_dir, - apr_psprintf(iterpool, - "%" APR_INT64_T_FMT PATH_EXT_PACKED_SHARD, - shard), - iterpool); - revprops_shard_path = svn_dirent_join(revsprops_dir, - apr_psprintf(iterpool, "%" APR_INT64_T_FMT, shard), - iterpool); - - SVN_ERR(svn_fs_x__pack_revprops_shard(revprops_pack_file_dir, - revprops_shard_path, - shard, ffd->max_files_per_dir, - (int)(0.9 * ffd->revprop_pack_size), - compression_level, - cancel_func, cancel_baton, iterpool)); - if (notify_func) - SVN_ERR(notify_func(notify_baton, shard, - svn_fs_upgrade_pack_revprops, iterpool)); - } - - svn_pool_destroy(iterpool); - - return SVN_NO_ERROR; -} - -svn_error_t * -svn_fs_x__upgrade_cleanup_pack_revprops(svn_fs_t *fs, - svn_fs_upgrade_notify_t notify_func, - void *notify_baton, - svn_cancel_func_t cancel_func, - void *cancel_baton, - apr_pool_t *scratch_pool) -{ - svn_fs_x__data_t *ffd = fs->fsap_data; - const char *revprops_shard_path; - apr_int64_t shard; - apr_int64_t first_unpacked_shard - = ffd->min_unpacked_rev / ffd->max_files_per_dir; - - apr_pool_t *iterpool = svn_pool_create(scratch_pool); - const char *revsprops_dir = svn_dirent_join(fs->path, PATH_REVPROPS_DIR, - scratch_pool); - - /* delete the non-packed revprops shards afterwards */ - for (shard = 0; shard < first_unpacked_shard; ++shard) - { - svn_pool_clear(iterpool); - - revprops_shard_path = svn_dirent_join(revsprops_dir, - apr_psprintf(iterpool, "%" APR_INT64_T_FMT, shard), - iterpool); - SVN_ERR(svn_fs_x__delete_revprops_shard(revprops_shard_path, - shard, ffd->max_files_per_dir, - cancel_func, cancel_baton, - iterpool)); - if (notify_func) - SVN_ERR(notify_func(notify_baton, shard, - svn_fs_upgrade_cleanup_revprops, iterpool)); - } - - svn_pool_destroy(iterpool); - - return SVN_NO_ERROR; -} /* Revprop caching management. * @@ -159,16 +67,7 @@ svn_fs_x__upgrade_cleanup_pack_revprops(svn_fs_t *fs, * as keys with the generation being incremented upon every revprop change. * Since the cache is process-local, the generation needs to be tracked * for at least as long as the process lives but may be reset afterwards. - * - * We track the revprop generation in a persistent, unbuffered file that - * we may keep open for the lifetime of the svn_fs_t. It is the OS' - * responsibility to provide us with the latest contents upon read. To - * detect incomplete updates due to non-atomic reads, we put a MD5 checksum - * next to the actual generation number and verify that it matches. - * - * Since we cannot guarantee that the OS will provide us with up-to-date - * data buffers for open files, we re-open and re-read the file before - * modifying it. This will prevent lost updates. + * We track the revprop generation in a file that. * * A race condition exists between switching to the modified revprop data * and bumping the generation number. In particular, the process may crash @@ -187,110 +86,6 @@ svn_fs_x__upgrade_cleanup_pack_revprops(svn_fs_t *fs, * after the crash, reader caches may be stale. */ -/* If the revprop generation file in FS is open, close it. This is a no-op - * if the file is not open. - */ -static svn_error_t * -close_revprop_generation_file(svn_fs_t *fs, - apr_pool_t *scratch_pool) -{ - svn_fs_x__data_t *ffd = fs->fsap_data; - if (ffd->revprop_generation_file) - { - SVN_ERR(svn_io_file_close(ffd->revprop_generation_file, scratch_pool)); - ffd->revprop_generation_file = NULL; - } - - return SVN_NO_ERROR; -} - -/* Make sure the revprop_generation member in FS is set. If READ_ONLY is - * set, open the file w/o write permission if the file is not open yet. - * The file is kept open if it has sufficient rights (or more) but will be - * closed and re-opened if it provided insufficient access rights. - * - * Call only for repos that support revprop caching. - */ -static svn_error_t * -open_revprop_generation_file(svn_fs_t *fs, - svn_boolean_t read_only, - apr_pool_t *scratch_pool) -{ - svn_fs_x__data_t *ffd = fs->fsap_data; - apr_int32_t flags = read_only ? APR_READ : (APR_READ | APR_WRITE); - - /* Close the current file handle if it has insufficient rights. */ - if ( ffd->revprop_generation_file - && (apr_file_flags_get(ffd->revprop_generation_file) & flags) != flags) - SVN_ERR(close_revprop_generation_file(fs, scratch_pool)); - - /* If not open already, open with sufficient rights. */ - if (ffd->revprop_generation_file == NULL) - { - const char *path = svn_fs_x__path_revprop_generation(fs, scratch_pool); - SVN_ERR(svn_io_file_open(&ffd->revprop_generation_file, path, - flags, APR_OS_DEFAULT, fs->pool)); - } - - return SVN_NO_ERROR; -} - -/* Return the textual representation of NUMBER and its checksum in *BUFFER. - */ -static svn_error_t * -checkedsummed_number(svn_stringbuf_t **buffer, - apr_int64_t number, - apr_pool_t *result_pool, - apr_pool_t *scratch_pool) -{ - svn_checksum_t *checksum; - const char *digest; - - char str[SVN_INT64_BUFFER_SIZE]; - apr_size_t len = svn__i64toa(str, number); - str[len] = 0; - - SVN_ERR(svn_checksum(&checksum, svn_checksum_md5, str, len, scratch_pool)); - digest = svn_checksum_to_cstring_display(checksum, scratch_pool); - - *buffer = svn_stringbuf_createf(result_pool, "%s %s\n", digest, str); - - return SVN_NO_ERROR; -} - -/* Extract the generation number from the text BUFFER of LEN bytes and - * verify it against the checksum in the same BUFFER. If they match, return - * the generation in *NUMBER. Otherwise, return an error. - * BUFFER does not need to be NUL-terminated. - */ -static svn_error_t * -verify_extract_number(apr_int64_t *number, - const char *buffer, - apr_size_t len, - apr_pool_t *scratch_pool) -{ - const char *digest_end = strchr(buffer, ' '); - - /* Does the buffer even contain checksum _and_ number? */ - if (digest_end != NULL) - { - svn_checksum_t *expected; - svn_checksum_t *actual; - - SVN_ERR(svn_checksum_parse_hex(&expected, svn_checksum_md5, buffer, - scratch_pool)); - SVN_ERR(svn_checksum(&actual, svn_checksum_md5, digest_end + 1, - (buffer + len) - (digest_end + 1), scratch_pool)); - - if (svn_checksum_match(expected, actual)) - return svn_error_trace(svn_cstring_atoi64(number, digest_end + 1)); - } - - /* Incomplete buffer or not a match. */ - return svn_error_create(SVN_ERR_FS_INVALID_GENERATION, NULL, - _("Invalid generation number data.")); -} - /* Read revprop generation as stored on disk for repository FS. The result is * returned in *CURRENT. Call only for repos that support revprop caching. */ @@ -299,40 +94,32 @@ read_revprop_generation_file(apr_int64_t *current, svn_fs_t *fs, apr_pool_t *scratch_pool) { - svn_fs_x__data_t *ffd = fs->fsap_data; apr_pool_t *iterpool = svn_pool_create(scratch_pool); - char buf[CHECKSUMMED_NUMBER_BUFFER_LEN]; - apr_size_t len; - apr_off_t offset = 0; int i; svn_error_t *err = SVN_NO_ERROR; + const char *path = svn_fs_x__path_revprop_generation(fs, scratch_pool); /* Retry in case of incomplete file buffer updates. */ for (i = 0; i < GENERATION_READ_RETRY_COUNT; ++i) { + svn_stringbuf_t *buf; + svn_error_clear(err); svn_pool_clear(iterpool); - /* If we can't even access the data, things are very wrong. - * Don't retry in that case. - */ - SVN_ERR(open_revprop_generation_file(fs, TRUE, iterpool)); - SVN_ERR(svn_io_file_seek(ffd->revprop_generation_file, APR_SET, &offset, - iterpool)); - - len = sizeof(buf); - SVN_ERR(svn_io_read_length_line(ffd->revprop_generation_file, buf, &len, - iterpool)); + /* Read the generation file. */ + err = svn_stringbuf_from_file2(&buf, path, iterpool); - /* Some data has been read. It will most likely be complete and - * consistent. Extract and verify anyway. */ - err = verify_extract_number(current, buf, len, iterpool); + /* If we could read the file, it should be complete due to our atomic + * file replacement scheme. */ if (!err) - break; - - /* Got unlucky and data was invalid. Retry. */ - SVN_ERR(close_revprop_generation_file(fs, iterpool)); + { + svn_stringbuf_strip_whitespace(buf); + SVN_ERR(svn_cstring_atoi64(current, buf->data)); + break; + } + /* Got unlucky the file was not available. Retry. */ #if APR_HAS_THREADS apr_thread_yield(); #else @@ -356,17 +143,21 @@ write_revprop_generation_file(svn_fs_t *fs, { svn_fs_x__data_t *ffd = fs->fsap_data; svn_stringbuf_t *buffer; - apr_off_t offset = 0; + const char *path = svn_fs_x__path_revprop_generation(fs, scratch_pool); - SVN_ERR(checkedsummed_number(&buffer, current, scratch_pool, scratch_pool)); + /* Invalidate our cached revprop generation in case the file operations + * below fail. */ + ffd->revprop_generation = -1; - SVN_ERR(open_revprop_generation_file(fs, FALSE, scratch_pool)); - SVN_ERR(svn_io_file_seek(ffd->revprop_generation_file, APR_SET, &offset, - scratch_pool)); - SVN_ERR(svn_io_file_write_full(ffd->revprop_generation_file, buffer->data, - buffer->len, NULL, scratch_pool)); - SVN_ERR(svn_io_file_flush_to_disk(ffd->revprop_generation_file, - scratch_pool)); + /* Write the new number. */ + buffer = svn_stringbuf_createf(scratch_pool, "%" APR_INT64_T_FMT "\n", + current); + SVN_ERR(svn_io_write_atomic2(path, buffer->data, buffer->len, + path /* copy_perms */, FALSE, + scratch_pool)); + + /* Remember it to spare us the re-read. */ + ffd->revprop_generation = current; return SVN_NO_ERROR; } @@ -375,49 +166,12 @@ svn_error_t * svn_fs_x__reset_revprop_generation_file(svn_fs_t *fs, apr_pool_t *scratch_pool) { - const char *path = svn_fs_x__path_revprop_generation(fs, scratch_pool); - svn_stringbuf_t *buffer; - - /* Unconditionally close the revprop generation file. - * Don't care about FS formats. This ensures consistent internal state. */ - SVN_ERR(close_revprop_generation_file(fs, scratch_pool)); - - /* Unconditionally remove any old revprop generation file. - * Don't care about FS formats. This ensures consistent on-disk state - * for old format repositories. */ - SVN_ERR(svn_io_remove_file2(path, TRUE, scratch_pool)); - - /* Write the initial revprop generation file contents, if supported by - * the current format. This ensures consistent on-disk state for new - * format repositories. */ - SVN_ERR(checkedsummed_number(&buffer, 0, scratch_pool, scratch_pool)); - SVN_ERR(svn_io_write_atomic(path, buffer->data, buffer->len, NULL, - scratch_pool)); - - /* ffd->revprop_generation_file will be re-opened on demand. */ + /* Write the initial revprop generation file contents. */ + SVN_ERR(write_revprop_generation_file(fs, 0, scratch_pool)); return SVN_NO_ERROR; } -/* Create an error object with the given MESSAGE and pass it to the - WARNING member of FS. Clears UNDERLYING_ERR. */ -static void -log_revprop_cache_init_warning(svn_fs_t *fs, - svn_error_t *underlying_err, - const char *message, - apr_pool_t *scratch_pool) -{ - svn_error_t *err = svn_error_createf( - SVN_ERR_FS_REVPROP_CACHE_INIT_FAILURE, - underlying_err, message, - svn_dirent_local_style(fs->path, scratch_pool)); - - if (fs->warning) - (fs->warning)(fs->warning_baton, err); - - svn_error_clear(err); -} - /* Test whether revprop cache and necessary infrastructure are available in FS. */ static svn_boolean_t @@ -425,29 +179,9 @@ has_revprop_cache(svn_fs_t *fs, apr_pool_t *scratch_pool) { svn_fs_x__data_t *ffd = fs->fsap_data; - svn_error_t *error; - - /* is the cache (still) enabled? */ - if (ffd->revprop_cache == NULL) - return FALSE; - - /* try initialize our file-backed infrastructure */ - error = open_revprop_generation_file(fs, TRUE, scratch_pool); - if (error) - { - /* failure -> disable revprop cache for good */ - - ffd->revprop_cache = NULL; - log_revprop_cache_init_warning(fs, error, - "Revprop caching for '%s' disabled " - "because infrastructure for revprop " - "caching failed to initialize.", - scratch_pool); - return FALSE; - } - - return TRUE; + /* is the cache enabled? */ + return ffd->revprop_cache != NULL; } /* Baton structure for revprop_generation_fixup. */ @@ -475,9 +209,6 @@ revprop_generation_fixup(void *void_baton, svn_fs_x__data_t *ffd = baton->fs->fsap_data; assert(ffd->has_write_lock); - /* Make sure we don't operate on stale OS buffers. */ - SVN_ERR(close_revprop_generation_file(baton->fs, scratch_pool)); - /* Maybe, either the original revprop writer or some other reader has already corrected / bumped the revprop generation. Thus, we need to read it again. However, we will now be the only ones changing @@ -498,12 +229,10 @@ revprop_generation_fixup(void *void_baton, return SVN_NO_ERROR; } -/* Read the current revprop generation and return it in *GENERATION. - Also, detect aborted / crashed writers and recover from that. - Use the access object in FS to set the shared mem values. */ +/* Read the current revprop generation of FS and its value in FS->FSAP_DATA. + Also, detect aborted / crashed writers and recover from that. */ static svn_error_t * -read_revprop_generation(apr_int64_t *generation, - svn_fs_t *fs, +read_revprop_generation(svn_fs_t *fs, apr_pool_t *scratch_pool) { apr_int64_t current = 0; @@ -548,60 +277,84 @@ read_revprop_generation(apr_int64_t *generation, } /* return the value we just got */ - *generation = current; + ffd->revprop_generation = current; return SVN_NO_ERROR; } +void +svn_fs_x__invalidate_revprop_generation(svn_fs_t *fs) +{ + svn_fs_x__data_t *ffd = fs->fsap_data; + ffd->revprop_generation = -1; +} + +/* Return TRUE if the revprop generation value in FS->FSAP_DATA is valid. */ +static svn_boolean_t +is_generation_valid(svn_fs_t *fs) +{ + svn_fs_x__data_t *ffd = fs->fsap_data; + return ffd->revprop_generation >= 0; +} + /* Set the revprop generation in FS to the next odd number to indicate - that there is a revprop write process under way. Return that value - in *GENERATION. If the change times out, readers shall recover from - that state & re-read revprops. + that there is a revprop write process under way. Update the value + in FS->FSAP_DATA accordingly. If the change times out, readers shall + recover from that state & re-read revprops. This is a no-op for repo formats that don't support revprop caching. */ static svn_error_t * -begin_revprop_change(apr_int64_t *generation, - svn_fs_t *fs, +begin_revprop_change(svn_fs_t *fs, apr_pool_t *scratch_pool) { svn_fs_x__data_t *ffd = fs->fsap_data; SVN_ERR_ASSERT(ffd->has_write_lock); - /* Close and re-open to make sure we read the latest data. */ - SVN_ERR(close_revprop_generation_file(fs, scratch_pool)); - SVN_ERR(open_revprop_generation_file(fs, FALSE, scratch_pool)); - /* Set the revprop generation to an odd value to indicate * that a write is in progress. */ - SVN_ERR(read_revprop_generation(generation, fs, scratch_pool)); - ++*generation; - SVN_ERR(write_revprop_generation_file(fs, *generation, scratch_pool)); + SVN_ERR(read_revprop_generation(fs, scratch_pool)); + ++ffd->revprop_generation; + SVN_ERR_ASSERT(ffd->revprop_generation % 2); + SVN_ERR(write_revprop_generation_file(fs, ffd->revprop_generation, + scratch_pool)); return SVN_NO_ERROR; } /* Set the revprop generation in FS to the next even generation after - the odd value in GENERATION to indicate that + the odd value in FS->FSAP_DATA to indicate that a) readers shall re-read revprops, and b) the write process has been completed (no recovery required). This is a no-op for repo formats that don't support revprop caching. */ static svn_error_t * end_revprop_change(svn_fs_t *fs, - apr_int64_t generation, apr_pool_t *scratch_pool) { svn_fs_x__data_t *ffd = fs->fsap_data; SVN_ERR_ASSERT(ffd->has_write_lock); - SVN_ERR_ASSERT(generation % 2); + SVN_ERR_ASSERT(ffd->revprop_generation % 2); /* Set the revprop generation to an even value to indicate * that a write has been completed. Since we held the write * lock, nobody else could have updated the file contents. */ - SVN_ERR(write_revprop_generation_file(fs, generation + 1, scratch_pool)); + SVN_ERR(write_revprop_generation_file(fs, ffd->revprop_generation + 1, + scratch_pool)); return SVN_NO_ERROR; } +/* Represents an entry in the packed revprop manifest. + * There is one such entry per pack file. */ +typedef struct manifest_entry_t +{ + /* First revision in the pack file. */ + svn_revnum_t start_rev; + + /* Tag (a counter) appended to the file name to distinguish it from + outdated ones. */ + apr_uint64_t tag; +} manifest_entry_t; + /* Container for all data required to access the packed revprop file * for a given REVISION. This structure will be filled incrementally * by read_pack_revprops() its sub-routines. @@ -611,9 +364,6 @@ typedef struct packed_revprops_t /* revision number to read (not necessarily the first in the pack) */ svn_revnum_t revision; - /* current revprop generation. Used when populating the revprop cache */ - apr_int64_t generation; - /* the actual revision properties */ apr_hash_t *properties; @@ -622,8 +372,8 @@ typedef struct packed_revprops_t apr_size_t serialized_size; - /* name of the pack file (without folder path) */ - const char *filename; + /* manifest entry describing the pack file */ + manifest_entry_t entry; /* packed shard folder path */ const char *folder; @@ -631,34 +381,19 @@ typedef struct packed_revprops_t /* sum of values in SIZES */ apr_size_t total_size; - /* first revision in the pack (>= MANIFEST_START) */ - svn_revnum_t start_revision; - - /* size of the revprops in PACKED_REVPROPS */ - apr_array_header_t *sizes; - - /* offset of the revprops in PACKED_REVPROPS */ - apr_array_header_t *offsets; - - - /* concatenation of the serialized representation of all revprops - * in the pack, i.e. the pack content without header and compression */ - svn_stringbuf_t *packed_revprops; - - /* First revision covered by MANIFEST. - * Will equal the shard start revision or 1, for the 1st shard. */ - svn_revnum_t manifest_start; + /* Array of svn_string_t, containing the serialized revprops for + * REVISION * I. */ + apr_array_header_t *revprops; /* content of the manifest. - * Maps long(rev - MANIFEST_START) to const char* pack file name */ + * Sorted list of manifest_entry_t. */ apr_array_header_t *manifest; } packed_revprops_t; /* Parse the serialized revprops in CONTENT and return them in *PROPERTIES. * Also, put them into the revprop cache, if activated, for future use. * Three more parameters are being used to update the revprop cache: FS is - * our file system, the revprops belong to REVISION and the global revprop - * GENERATION is used as well. + * our file system, the revprops belong to REVISION. * * The returned hash will be allocated in RESULT_POOL, SCRATCH_POOL is * being used for temporary allocations. @@ -667,23 +402,23 @@ static svn_error_t * parse_revprop(apr_hash_t **properties, svn_fs_t *fs, svn_revnum_t revision, - apr_int64_t generation, - svn_string_t *content, + const svn_string_t *content, apr_pool_t *result_pool, apr_pool_t *scratch_pool) { - svn_stream_t *stream = svn_stream_from_string(content, scratch_pool); - *properties = apr_hash_make(result_pool); + SVN_ERR_W(svn_fs_x__parse_properties(properties, content, result_pool), + apr_psprintf(scratch_pool, "Failed to parse revprops for r%ld.", + revision)); - SVN_ERR(svn_hash_read2(*properties, stream, SVN_HASH_TERMINATOR, - result_pool)); if (has_revprop_cache(fs, scratch_pool)) { svn_fs_x__data_t *ffd = fs->fsap_data; svn_fs_x__pair_cache_key_t key = { 0 }; + SVN_ERR_ASSERT(is_generation_valid(fs)); + key.revision = revision; - key.second = generation; + key.second = ffd->revprop_generation; SVN_ERR(svn_cache__set(ffd->revprop_cache, &key, *properties, scratch_pool)); } @@ -691,9 +426,37 @@ parse_revprop(apr_hash_t **properties, return SVN_NO_ERROR; } +/* Verify the checksum attached to CONTENT and remove it. + * Use SCRATCH_POOL for temporary allocations. + */ +static svn_error_t * +verify_checksum(svn_stringbuf_t *content, + apr_pool_t *scratch_pool) +{ + const apr_byte_t *digest; + svn_checksum_t *actual, *expected; + + /* Verify the checksum. */ + if (content->len < sizeof(apr_uint32_t)) + return svn_error_create(SVN_ERR_CORRUPT_PACKED_DATA, NULL, + "File too short"); + + content->len -= sizeof(apr_uint32_t); + digest = (apr_byte_t *)content->data + content->len; + + expected = svn_checksum__from_digest_fnv1a_32x4(digest, scratch_pool); + SVN_ERR(svn_checksum(&actual, svn_checksum_fnv1a_32x4, content->data, + content->len, scratch_pool)); + + if (!svn_checksum_match(actual, expected)) + SVN_ERR(svn_checksum_mismatch_err(expected, actual, scratch_pool, + "checksum mismatch")); + + return SVN_NO_ERROR; +} + /* Read the non-packed revprops for revision REV in FS, put them into the - * revprop cache if activated and return them in *PROPERTIES. GENERATION - * is the current revprop generation. + * revprop cache if activated and return them in *PROPERTIES. * * If the data could not be read due to an otherwise recoverable error, * leave *PROPERTIES unchanged. No error will be returned in that case. @@ -704,7 +467,6 @@ static svn_error_t * read_non_packed_revprop(apr_hash_t **properties, svn_fs_t *fs, svn_revnum_t rev, - apr_int64_t generation, apr_pool_t *result_pool, apr_pool_t *scratch_pool) { @@ -726,26 +488,193 @@ read_non_packed_revprop(apr_hash_t **properties, } if (content) - SVN_ERR(parse_revprop(properties, fs, rev, generation, - svn_stringbuf__morph_into_string(content), - result_pool, iterpool)); + { + svn_string_t *as_string; + + /* Consistency check. */ + SVN_ERR_W(verify_checksum(content, scratch_pool), + apr_psprintf(scratch_pool, + "Revprop file for r%ld is corrupt", + rev)); + + /* The contents string becomes part of the *PROPERTIES structure, i.e. + * we must make sure it lives at least as long as the latter. */ + as_string = svn_string_create_from_buf(content, result_pool); + SVN_ERR(parse_revprop(properties, fs, rev, as_string, + result_pool, iterpool)); + } svn_pool_clear(iterpool); return SVN_NO_ERROR; } -/* Return the minimum length of any packed revprop file name in REVPROPS. */ -static apr_size_t -get_min_filename_len(packed_revprops_t *revprops) +/* Serialize ROOT into FILE and append a checksum to it. + * Use SCRATCH_POOL for temporary allocations. + */ +static svn_error_t * +write_packed_data_checksummed(svn_packed__data_root_t *root, + apr_file_t *file, + apr_pool_t *scratch_pool) { - char number_buffer[SVN_INT64_BUFFER_SIZE]; + svn_checksum_t *checksum; + svn_stream_t *stream; - /* The revprop filenames have the format <REV>.<COUNT> - with <REV> being - * at least the first rev in the shard and <COUNT> having at least one - * digit. Thus, the minimum is 2 + #decimal places in the start rev. - */ - return svn__i64toa(number_buffer, revprops->manifest_start) + 2; + stream = svn_stream_from_aprfile2(file, TRUE, scratch_pool); + stream = svn_checksum__wrap_write_stream(&checksum, stream, + svn_checksum_fnv1a_32x4, + scratch_pool); + SVN_ERR(svn_packed__data_write(stream, root, scratch_pool)); + SVN_ERR(svn_stream_close(stream)); + + /* Append the checksum */ + SVN_ERR(svn_io_file_write_full(file, checksum->digest, + svn_checksum_size(checksum), NULL, + scratch_pool)); + + return SVN_NO_ERROR; +} + +/* Serialize the packed revprops MANIFEST into FILE. + * Use SCRATCH_POOL for temporary allocations. + */ +static svn_error_t * +write_manifest(apr_file_t *file, + const apr_array_header_t *manifest, + apr_pool_t *scratch_pool) +{ + int i; + svn_packed__data_root_t *root = svn_packed__data_create_root(scratch_pool); + + /* one top-level stream per struct element */ + svn_packed__int_stream_t *start_rev_stream + = svn_packed__create_int_stream(root, TRUE, FALSE); + svn_packed__int_stream_t *tag_stream + = svn_packed__create_int_stream(root, FALSE, FALSE); + + /* serialize ENTRIES */ + for (i = 0; i < manifest->nelts; ++i) + { + manifest_entry_t *entry = &APR_ARRAY_IDX(manifest, i, manifest_entry_t); + svn_packed__add_uint(start_rev_stream, entry->start_rev); + svn_packed__add_uint(tag_stream, entry->tag); + } + + /* Write to file and calculate the checksum. */ + SVN_ERR(write_packed_data_checksummed(root, file, scratch_pool)); + + return SVN_NO_ERROR; +} + +/* Read *ROOT from CONTENT and verify its checksum. Allocate *ROOT in + * RESULT_POOL and use SCRATCH_POOL for temporary allocations. + */ +static svn_error_t * +read_packed_data_checksummed(svn_packed__data_root_t **root, + svn_stringbuf_t *content, + apr_pool_t *result_pool, + apr_pool_t *scratch_pool) +{ + svn_stream_t *stream; + + SVN_ERR(verify_checksum(content, scratch_pool)); + + stream = svn_stream_from_stringbuf(content, scratch_pool); + SVN_ERR(svn_packed__data_read(root, stream, result_pool, scratch_pool)); + + return SVN_NO_ERROR; +} + +/* Read the packed revprops manifest from the CONTENT buffer and return it + * in *MANIFEST, allocated in RESULT_POOL. REVISION is the revision number + * to put into error messages. Use SCRATCH_POOL for temporary allocations. + */ +static svn_error_t * +read_manifest(apr_array_header_t **manifest, + svn_stringbuf_t *content, + svn_revnum_t revision, + apr_pool_t *result_pool, + apr_pool_t *scratch_pool) +{ + apr_size_t i; + apr_size_t count; + + svn_packed__data_root_t *root; + svn_packed__int_stream_t *start_rev_stream; + svn_packed__int_stream_t *tag_stream; + + /* Verify the checksum and decode packed data. */ + SVN_ERR_W(read_packed_data_checksummed(&root, content, result_pool, + scratch_pool), + apr_psprintf(scratch_pool, + "Revprop manifest file for r%ld is corrupt", + revision)); + + /* get streams */ + start_rev_stream = svn_packed__first_int_stream(root); + tag_stream = svn_packed__next_int_stream(start_rev_stream); + + /* read ids array */ + count = svn_packed__int_count(start_rev_stream); + *manifest = apr_array_make(result_pool, (int)count, + sizeof(manifest_entry_t)); + + for (i = 0; i < count; ++i) + { + manifest_entry_t *entry = apr_array_push(*manifest); + entry->start_rev = (svn_revnum_t)svn_packed__get_int(start_rev_stream); + entry->tag = svn_packed__get_uint(tag_stream); + } + + return SVN_NO_ERROR; +} + +/* Implements the standard comparison function signature comparing the + * manifest_entry_t(lhs).start_rev to svn_revnum_t(rhs). */ +static int +compare_entry_revision(const void *lhs, + const void *rhs) +{ + const manifest_entry_t *entry = lhs; + const svn_revnum_t *revision = rhs; + + if (entry->start_rev < *revision) + return -1; + + return entry->start_rev == *revision ? 0 : 1; +} + +/* Return the index in MANIFEST that has the info for the pack file + * containing REVISION. */ +static int +get_entry(apr_array_header_t *manifest, + svn_revnum_t revision) +{ + manifest_entry_t *entry; + int idx = svn_sort__bsearch_lower_bound(manifest, &revision, + compare_entry_revision); + + assert(manifest->nelts > 0); + if (idx >= manifest->nelts) + return idx - 1; + + entry = &APR_ARRAY_IDX(manifest, idx, manifest_entry_t); + if (entry->start_rev > revision && idx > 0) + return idx - 1; + + return idx; +} + +/* Return the full path of the revprop pack file given by ENTRY within + * REVPROPS. Allocate the result in RESULT_POOL. */ +static const char * +get_revprop_pack_filepath(packed_revprops_t *revprops, + manifest_entry_t *entry, + apr_pool_t *result_pool) +{ + const char *filename = apr_psprintf(result_pool, "%ld.%" APR_UINT64_T_FMT, + entry->start_rev, entry->tag); + return svn_dirent_join(revprops->folder, filename, result_pool); } /* Given FS and REVPROPS->REVISION, fill the FILENAME, FOLDER and MANIFEST @@ -761,98 +690,59 @@ get_revprop_packname(svn_fs_t *fs, svn_fs_x__data_t *ffd = fs->fsap_data; svn_stringbuf_t *content = NULL; const char *manifest_file_path; - int idx, rev_count; - char *buffer, *buffer_end; - const char **filenames, **filenames_end; - apr_size_t min_filename_len; + int idx; + svn_revnum_t previous_start_rev; + int i; /* Determine the dimensions. Rev 0 is excluded from the first shard. */ - rev_count = ffd->max_files_per_dir; - revprops->manifest_start + int rev_count = ffd->max_files_per_dir; + svn_revnum_t manifest_start = revprops->revision - (revprops->revision % rev_count); - if (revprops->manifest_start == 0) + if (manifest_start == 0) { - ++revprops->manifest_start; + ++manifest_start; --rev_count; } - revprops->manifest = apr_array_make(result_pool, rev_count, - sizeof(const char*)); - - /* No line in the file can be less than this number of chars long. */ - min_filename_len = get_min_filename_len(revprops); - /* Read the content of the manifest file */ - revprops->folder - = svn_fs_x__path_revprops_pack_shard(fs, revprops->revision, result_pool); + revprops->folder = svn_fs_x__path_pack_shard(fs, revprops->revision, + result_pool); manifest_file_path = svn_dirent_join(revprops->folder, PATH_MANIFEST, result_pool); - SVN_ERR(svn_fs_x__read_content(&content, manifest_file_path, result_pool)); + SVN_ERR(read_manifest(&revprops->manifest, content, revprops->revision, + result_pool, scratch_pool)); - /* There CONTENT must have a certain minimal size and there no - * unterminated lines at the end of the file. Both guarantees also - * simplify the parser loop below. - */ - if ( content->len < rev_count * (min_filename_len + 1) - || content->data[content->len - 1] != '\n') - return svn_error_createf(SVN_ERR_FS_CORRUPT, NULL, - _("Packed revprop manifest for r%ld not " - "properly terminated"), revprops->revision); - - /* Chop (parse) the manifest CONTENT into filenames, one per line. - * We only have to replace all newlines with NUL and add all line - * starts to REVPROPS->MANIFEST. - * - * There must be exactly REV_COUNT lines and that is the number of - * lines we parse from BUFFER to FILENAMES. Set the end pointer for - * the source BUFFER such that BUFFER+MIN_FILENAME_LEN is still valid - * BUFFER_END is always valid due to CONTENT->LEN > MIN_FILENAME_LEN. - * - * Please note that this loop is performance critical for e.g. 'svn log'. - * It is run 1000x per revprop access, i.e. per revision and about - * 50 million times per sec (and CPU core). - */ - for (filenames = (const char **)revprops->manifest->elts, - filenames_end = filenames + rev_count, - buffer = content->data, - buffer_end = buffer + content->len - min_filename_len; - (filenames < filenames_end) && (buffer < buffer_end); - ++filenames) + /* Verify the manifest data. */ + if (revprops->manifest->nelts == 0) + return svn_error_createf(SVN_ERR_FS_CORRUPT_REVPROP_MANIFEST, NULL, + "Revprop manifest for r%ld is empty", + revprops->revision); + + previous_start_rev = 0; + for (i = 0; i < revprops->manifest->nelts; ++i) { - /* BUFFER always points to the start of the next line / filename. */ - *filenames = buffer; - - /* Find the next EOL. This is guaranteed to stay within the CONTENT - * buffer because we left enough room after BUFFER_END and we know - * we will always see a newline as the last non-NUL char. */ - buffer += min_filename_len; - while (*buffer != '\n') - ++buffer; - - /* Found EOL. Turn it into the filename terminator and move BUFFER - * to the start of the next line or CONTENT buffer end. */ - *buffer = '\0'; - ++buffer; + svn_revnum_t start_rev = APR_ARRAY_IDX(revprops->manifest, i, + manifest_entry_t).start_rev; + if ( start_rev < manifest_start + || start_rev >= manifest_start + rev_count) + return svn_error_createf(SVN_ERR_FS_CORRUPT_REVPROP_MANIFEST, NULL, + "Revprop manifest for r%ld contains " + "out-of-range revision r%ld", + revprops->revision, start_rev); + + if (start_rev < previous_start_rev) + return svn_error_createf(SVN_ERR_FS_CORRUPT_REVPROP_MANIFEST, NULL, + "Entries in revprop manifest for r%ld " + "are not ordered", revprops->revision); + + previous_start_rev = start_rev; } - /* We must have reached the end of both buffers. */ - if (buffer < content->data + content->len) - return svn_error_createf(SVN_ERR_FS_CORRUPT, NULL, - _("Packed revprop manifest for r%ld " - "has too many entries"), revprops->revision); - - if (filenames < filenames_end) - return svn_error_createf(SVN_ERR_FS_CORRUPT, NULL, - _("Packed revprop manifest for r%ld " - "has too few entries"), revprops->revision); - - /* The target array has now exactly one entry per revision. */ - revprops->manifest->nelts = rev_count; - - /* Now get the file name */ - idx = (int)(revprops->revision - revprops->manifest_start); - revprops->filename = APR_ARRAY_IDX(revprops->manifest, idx, const char*); + /* Now get the pack file description */ + idx = get_entry(revprops->manifest, revprops->revision); + revprops->entry = APR_ARRAY_IDX(revprops->manifest, idx, + manifest_entry_t); return SVN_NO_ERROR; } @@ -868,10 +758,9 @@ same_shard(svn_fs_t *fs, return (r1 / ffd->max_files_per_dir) == (r2 / ffd->max_files_per_dir); } -/* Given FS and the full packed file content in REVPROPS->PACKED_REVPROPS, - * fill the START_REVISION member, and make PACKED_REVPROPS point to the - * first serialized revprop. If READ_ALL is set, initialize the SIZES - * and OFFSETS members as well. +/* Given FS and the full packed file content in CONTENT and make + * PACKED_REVPROPS point to the first serialized revprop. If READ_ALL + * is set, initialize the SIZES and OFFSETS members as well. * * Parse the revprops for REVPROPS->REVISION and set the PROPERTIES as * well as the SERIALIZED_SIZE member. If revprop caching has been @@ -880,33 +769,31 @@ same_shard(svn_fs_t *fs, static svn_error_t * parse_packed_revprops(svn_fs_t *fs, packed_revprops_t *revprops, + svn_stringbuf_t *content, svn_boolean_t read_all, apr_pool_t *result_pool, apr_pool_t *scratch_pool) { - svn_stream_t *stream; - apr_int64_t first_rev, count, i; - apr_off_t offset; - const char *header_end; + apr_size_t count, i; apr_pool_t *iterpool = svn_pool_create(scratch_pool); svn_boolean_t cache_all = has_revprop_cache(fs, scratch_pool); + svn_packed__data_root_t *root; + svn_packed__byte_stream_t *revprops_stream; + svn_revnum_t first_rev = revprops->entry.start_rev; - /* decompress (even if the data is only "stored", there is still a - * length header to remove) */ - svn_stringbuf_t *compressed = revprops->packed_revprops; - svn_stringbuf_t *uncompressed = svn_stringbuf_create_empty(result_pool); - SVN_ERR(svn__decompress(compressed, uncompressed, APR_SIZE_MAX)); + /* Verify the checksum and decode packed data. */ + SVN_ERR_W(read_packed_data_checksummed(&root, content, result_pool, + scratch_pool), + apr_psprintf(scratch_pool, + "Revprop pack file for r%ld is corrupt", + first_rev)); - /* read first revision number and number of revisions in the pack */ - stream = svn_stream_from_stringbuf(uncompressed, scratch_pool); - SVN_ERR(svn_fs_x__read_number_from_stream(&first_rev, NULL, stream, - iterpool)); - SVN_ERR(svn_fs_x__read_number_from_stream(&count, NULL, stream, iterpool)); + /* get streams */ + revprops_stream = svn_packed__first_byte_stream(root); + count = svn_packed__byte_block_count(revprops_stream); /* Check revision range for validity. */ - if ( !same_shard(fs, revprops->revision, first_rev) - || !same_shard(fs, revprops->revision, first_rev + count - 1) - || count < 1) + if (!same_shard(fs, first_rev, first_rev + count - 1) || count < 1) return svn_error_createf(SVN_ERR_FS_CORRUPT, NULL, _("Revprop pack for revision r%ld" " contains revprops for r%ld .. r%ld"), @@ -923,58 +810,35 @@ parse_packed_revprops(svn_fs_t *fs, " starts at non-packed revisions r%ld"), revprops->revision, (svn_revnum_t)first_rev); - /* make PACKED_REVPROPS point to the first char after the header. - * This is where the serialized revprops are. */ - header_end = strstr(uncompressed->data, "\n\n"); - if (header_end == NULL) - return svn_error_create(SVN_ERR_FS_CORRUPT, NULL, - _("Header end not found")); - - offset = header_end - uncompressed->data + 2; - - revprops->packed_revprops = svn_stringbuf_create_empty(result_pool); - revprops->packed_revprops->data = uncompressed->data + offset; - revprops->packed_revprops->len = (apr_size_t)(uncompressed->len - offset); - revprops->packed_revprops->blocksize = (apr_size_t)(uncompressed->blocksize - offset); - - /* STREAM still points to the first entry in the sizes list. */ - revprops->start_revision = (svn_revnum_t)first_rev; - if (read_all) + /* Request all data (just references to data already expanded in ROOT) */ + revprops->revprops = apr_array_make(result_pool, (int)count, + sizeof(svn_string_t)); + for (i = 0, revprops->total_size = 0; i < count; ++i) { - /* Init / construct REVPROPS members. */ - revprops->sizes = apr_array_make(result_pool, (int)count, - sizeof(offset)); - revprops->offsets = apr_array_make(result_pool, (int)count, - sizeof(offset)); + svn_string_t *props = apr_array_push(revprops->revprops); + props->data = svn_packed__get_bytes(revprops_stream, &props->len); + + revprops->total_size += props->len; } - /* Now parse, revision by revision, the size and content of each - * revisions' revprops. */ - for (i = 0, offset = 0, revprops->total_size = 0; i < count; ++i) + /* Now parse the serialized revprops. */ + for (i = 0; i < count; ++i) { - apr_int64_t size; - svn_string_t serialized; - svn_revnum_t revision = (svn_revnum_t)(first_rev + i); + const svn_string_t *serialized; + svn_revnum_t revision; + svn_pool_clear(iterpool); - /* read & check the serialized size */ - SVN_ERR(svn_fs_x__read_number_from_stream(&size, NULL, stream, - iterpool)); - if (size + offset > (apr_int64_t)revprops->packed_revprops->len) - return svn_error_create(SVN_ERR_FS_CORRUPT, NULL, - _("Packed revprop size exceeds pack file size")); + serialized = &APR_ARRAY_IDX(revprops->revprops, (int)i, svn_string_t); + revision = first_rev + (long)i; /* Parse this revprops list, if necessary */ - serialized.data = revprops->packed_revprops->data + offset; - serialized.len = (apr_size_t)size; - if (revision == revprops->revision) { /* Parse (and possibly cache) the one revprop list we care about. */ SVN_ERR(parse_revprop(&revprops->properties, fs, revision, - revprops->generation, &serialized, - result_pool, iterpool)); - revprops->serialized_size = serialized.len; + serialized, result_pool, iterpool)); + revprops->serialized_size = serialized->len; /* If we only wanted the revprops for REVISION then we are done. */ if (!read_all && !cache_all) @@ -984,29 +848,20 @@ parse_packed_revprops(svn_fs_t *fs, { /* Parse and cache all other revprop lists. */ apr_hash_t *properties; - SVN_ERR(parse_revprop(&properties, fs, revision, - revprops->generation, &serialized, + SVN_ERR(parse_revprop(&properties, fs, revision, serialized, iterpool, iterpool)); } - - if (read_all) - { - /* fill REVPROPS data structures */ - APR_ARRAY_PUSH(revprops->sizes, apr_off_t) = serialized.len; - APR_ARRAY_PUSH(revprops->offsets, apr_off_t) = offset; - } - revprops->total_size += serialized.len; - - offset += serialized.len; } + svn_pool_destroy(iterpool); + return SVN_NO_ERROR; } /* In filesystem FS, read the packed revprops for revision REV into - * *REVPROPS. Use GENERATION to populate the revprop cache, if enabled. - * If you want to modify revprop contents / update REVPROPS, READ_ALL - * must be set. Otherwise, only the properties of REV are being provided. + * *REVPROPS. Populate the revprop cache, if enabled. If you want to + * modify revprop contents / update REVPROPS, READ_ALL must be set. + * Otherwise, only the properties of REV are being provided. * * Allocate *PROPERTIES in RESULT_POOL and temporaries in SCRATCH_POOL. */ @@ -1014,14 +869,12 @@ static svn_error_t * read_pack_revprop(packed_revprops_t **revprops, svn_fs_t *fs, svn_revnum_t rev, - apr_int64_t generation, svn_boolean_t read_all, apr_pool_t *result_pool, apr_pool_t *scratch_pool) { apr_pool_t *iterpool = svn_pool_create(scratch_pool); svn_boolean_t missing = FALSE; - svn_error_t *err; packed_revprops_t *result; int i; @@ -1036,29 +889,37 @@ read_pack_revprop(packed_revprops_t **revprops, /* initialize the result data structure */ result = apr_pcalloc(result_pool, sizeof(*result)); result->revision = rev; - result->generation = generation; /* try to read the packed revprops. This may require retries if we have * concurrent writers. */ - for (i = 0; - i < SVN_FS_X__RECOVERABLE_RETRY_COUNT && !result->packed_revprops; - ++i) + for (i = 0; i < SVN_FS_X__RECOVERABLE_RETRY_COUNT; ++i) { const char *file_path; + svn_stringbuf_t *contents = NULL; + svn_pool_clear(iterpool); /* there might have been concurrent writes. * Re-read the manifest and the pack file. */ SVN_ERR(get_revprop_packname(fs, result, result_pool, iterpool)); - file_path = svn_dirent_join(result->folder, - result->filename, - iterpool); - SVN_ERR(svn_fs_x__try_stringbuf_from_file(&result->packed_revprops, + file_path = get_revprop_pack_filepath(result, &result->entry, + iterpool); + SVN_ERR(svn_fs_x__try_stringbuf_from_file(&contents, &missing, file_path, i + 1 < SVN_FS_X__RECOVERABLE_RETRY_COUNT, - result_pool)); + iterpool)); + + if (contents) + { + SVN_ERR_W(parse_packed_revprops(fs, result, contents, read_all, + result_pool, iterpool), + apr_psprintf(iterpool, + "Revprop pack file for r%ld is corrupt", + rev)); + break; + } /* If we could not find the file, there was a write. * So, we should refresh our revprop generation info as well such @@ -1066,40 +927,29 @@ read_pack_revprop(packed_revprops_t **revprops, * consider it outdated, otherwise. */ if (missing && has_revprop_cache(fs, iterpool)) - SVN_ERR(read_revprop_generation(&result->generation, fs, iterpool)); + SVN_ERR(read_revprop_generation(fs, iterpool)); } /* the file content should be available now */ - if (!result->packed_revprops) + if (!result->revprops) return svn_error_createf(SVN_ERR_FS_PACKED_REVPROP_READ_FAILURE, NULL, _("Failed to read revprop pack file for r%ld"), rev); - /* parse it. RESULT will be complete afterwards. */ - err = parse_packed_revprops(fs, result, read_all, result_pool, iterpool); - svn_pool_destroy(iterpool); - if (err) - return svn_error_createf(SVN_ERR_FS_CORRUPT, err, - _("Revprop pack file for r%ld is corrupt"), rev); - *revprops = result; return SVN_NO_ERROR; } -/* Read the revprops for revision REV in FS and return them in *PROPERTIES_P. - * - * Allocations will be done in POOL. - */ svn_error_t * svn_fs_x__get_revision_proplist(apr_hash_t **proplist_p, svn_fs_t *fs, svn_revnum_t rev, svn_boolean_t bypass_cache, + svn_boolean_t refresh, apr_pool_t *result_pool, apr_pool_t *scratch_pool) { svn_fs_x__data_t *ffd = fs->fsap_data; - apr_int64_t generation = 0; /* not found, yet */ *proplist_p = NULL; @@ -1107,16 +957,18 @@ svn_fs_x__get_revision_proplist(apr_hash_t **proplist_p, /* should they be available at all? */ SVN_ERR(svn_fs_x__ensure_revision_exists(rev, fs, scratch_pool)); + /* Ensure that the revprop generation info is valid. */ + if (refresh || !is_generation_valid(fs)) + SVN_ERR(read_revprop_generation(fs, scratch_pool)); + /* Try cache lookup first. */ if (!bypass_cache && has_revprop_cache(fs, scratch_pool)) { svn_boolean_t is_cached; svn_fs_x__pair_cache_key_t key = { 0 }; - SVN_ERR(read_revprop_generation(&generation, fs, scratch_pool)); - key.revision = rev; - key.second = generation; + key.second = ffd->revprop_generation; SVN_ERR(svn_cache__get((void **) proplist_p, &is_cached, ffd->revprop_cache, &key, result_pool)); if (is_cached) @@ -1129,8 +981,7 @@ svn_fs_x__get_revision_proplist(apr_hash_t **proplist_p, if (!svn_fs_x__is_packed_revprop(fs, rev)) { svn_error_t *err = read_non_packed_revprop(proplist_p, fs, rev, - generation, result_pool, - scratch_pool); + result_pool, scratch_pool); if (err) { if (!APR_STATUS_IS_ENOENT(err->apr_err)) @@ -1147,7 +998,7 @@ svn_fs_x__get_revision_proplist(apr_hash_t **proplist_p, if (!*proplist_p) { packed_revprops_t *revprops; - SVN_ERR(read_pack_revprop(&revprops, fs, rev, generation, FALSE, + SVN_ERR(read_pack_revprop(&revprops, fs, rev, FALSE, result_pool, scratch_pool)); *proplist_p = revprops->properties; } @@ -1161,10 +1012,33 @@ svn_fs_x__get_revision_proplist(apr_hash_t **proplist_p, return SVN_NO_ERROR; } +svn_error_t * +svn_fs_x__write_non_packed_revprops(apr_file_t *file, + apr_hash_t *proplist, + apr_pool_t *scratch_pool) +{ + svn_stream_t *stream; + svn_checksum_t *checksum; + + stream = svn_stream_from_aprfile2(file, TRUE, scratch_pool); + stream = svn_checksum__wrap_write_stream(&checksum, stream, + svn_checksum_fnv1a_32x4, + scratch_pool); + SVN_ERR(svn_fs_x__write_properties(stream, proplist, scratch_pool)); + SVN_ERR(svn_stream_close(stream)); + + /* Append the checksum */ + SVN_ERR(svn_io_file_write_full(file, checksum->digest, + svn_checksum_size(checksum), NULL, + scratch_pool)); + + return SVN_NO_ERROR; +} + /* Serialize the revision property list PROPLIST of revision REV in * filesystem FS to a non-packed file. Return the name of that temporary * file in *TMP_PATH and the file path that it must be moved to in - * *FINAL_PATH. + * *FINAL_PATH. Schedule necessary fsync calls in BATCH. * * Allocate *FINAL_PATH and *TMP_PATH in RESULT_POOL. Use SCRATCH_POOL * for temporary allocations. @@ -1175,29 +1049,25 @@ write_non_packed_revprop(const char **final_path, svn_fs_t *fs, svn_revnum_t rev, apr_hash_t *proplist, + svn_fs_x__batch_fsync_t *batch, apr_pool_t *result_pool, apr_pool_t *scratch_pool) { - svn_stream_t *stream; + apr_file_t *file; *final_path = svn_fs_x__path_revprops(fs, rev, result_pool); - /* ### do we have a directory sitting around already? we really shouldn't - ### have to get the dirname here. */ - SVN_ERR(svn_stream_open_unique(&stream, tmp_path, - svn_dirent_dirname(*final_path, - scratch_pool), - svn_io_file_del_none, - result_pool, scratch_pool)); - SVN_ERR(svn_hash_write2(proplist, stream, SVN_HASH_TERMINATOR, - scratch_pool)); - SVN_ERR(svn_stream_close(stream)); + *tmp_path = apr_pstrcat(result_pool, *final_path, ".tmp", SVN_VA_NULL); + SVN_ERR(svn_fs_x__batch_fsync_open_file(&file, batch, *tmp_path, + scratch_pool)); + + SVN_ERR(svn_fs_x__write_non_packed_revprops(file, proplist, scratch_pool)); return SVN_NO_ERROR; } /* After writing the new revprop file(s), call this function to move the * file at TMP_PATH to FINAL_PATH and give it the permissions from - * PERMS_REFERENCE. + * PERMS_REFERENCE. Schedule necessary fsync calls in BATCH. * * If indicated in BUMP_GENERATION, increase FS' revprop generation. * Finally, delete all the temporary files given in FILES_TO_DELETE. @@ -1212,21 +1082,26 @@ switch_to_new_revprop(svn_fs_t *fs, const char *perms_reference, apr_array_header_t *files_to_delete, svn_boolean_t bump_generation, + svn_fs_x__batch_fsync_t *batch, apr_pool_t *scratch_pool) { - apr_int64_t generation; - /* Now, we may actually be replacing revprops. Make sure that all other threads and processes will know about this. */ if (bump_generation) - SVN_ERR(begin_revprop_change(&generation, fs, scratch_pool)); + SVN_ERR(begin_revprop_change(fs, scratch_pool)); + + /* Ensure the new file contents makes it to disk before switching over to + * it. */ + SVN_ERR(svn_fs_x__batch_fsync_run(batch, scratch_pool)); + /* Make the revision visible to all processes and threads. */ SVN_ERR(svn_fs_x__move_into_place(tmp_path, final_path, perms_reference, - scratch_pool)); + batch, scratch_pool)); + SVN_ERR(svn_fs_x__batch_fsync_run(batch, scratch_pool)); /* Indicate that the update (if relevant) has been completed. */ if (bump_generation) - SVN_ERR(end_revprop_change(fs, generation, scratch_pool)); + SVN_ERR(end_revprop_change(fs, scratch_pool)); /* Clean up temporary files, if necessary. */ if (files_to_delete) @@ -1247,52 +1122,8 @@ switch_to_new_revprop(svn_fs_t *fs, return SVN_NO_ERROR; } -/* Write a pack file header to STREAM that starts at revision START_REVISION - * and contains the indexes [START,END) of SIZES. - */ -static svn_error_t * -serialize_revprops_header(svn_stream_t *stream, - svn_revnum_t start_revision, - apr_array_header_t *sizes, - int start, - int end, - apr_pool_t *scratch_pool) -{ - apr_pool_t *iterpool = svn_pool_create(scratch_pool); - int i; - - SVN_ERR_ASSERT(start < end); - - /* start revision and entry count */ - SVN_ERR(svn_stream_printf(stream, scratch_pool, "%ld\n", start_revision)); - SVN_ERR(svn_stream_printf(stream, scratch_pool, "%d\n", end - start)); - - /* the sizes array */ - for (i = start; i < end; ++i) - { - /* Non-standard pool usage. - * - * We only allocate a few bytes each iteration -- even with a - * million iterations we would still be in good shape memory-wise. - */ - apr_off_t size = APR_ARRAY_IDX(sizes, i, apr_off_t); - SVN_ERR(svn_stream_printf(stream, iterpool, "%" APR_OFF_T_FMT "\n", - size)); - } - - /* the double newline char indicates the end of the header */ - SVN_ERR(svn_stream_printf(stream, iterpool, "\n")); - - svn_pool_destroy(iterpool); - return SVN_NO_ERROR; -} - -/* Writes the a pack file to FILE_STREAM. It copies the serialized data - * from REVPROPS for the indexes [START,END) except for index CHANGED_INDEX. - * - * The data for the latter is taken from NEW_SERIALIZED. Note, that - * CHANGED_INDEX may be outside the [START,END) range, i.e. no new data is - * taken in that case but only a subset of the old data will be copied. +/* Writes the a pack file to FILE. It copies the serialized data + * from REVPROPS for the indexes [START,END). * * NEW_TOTAL_SIZE is a hint for pre-allocating buffers of appropriate size. * SCRATCH_POOL is used for temporary allocations. @@ -1302,137 +1133,99 @@ repack_revprops(svn_fs_t *fs, packed_revprops_t *revprops, int start, int end, - int changed_index, - svn_stringbuf_t *new_serialized, - apr_off_t new_total_size, - svn_stream_t *file_stream, + apr_size_t new_total_size, + apr_file_t *file, apr_pool_t *scratch_pool) { - svn_fs_x__data_t *ffd = fs->fsap_data; - svn_stream_t *stream; int i; - /* create data empty buffers and the stream object */ - svn_stringbuf_t *uncompressed - = svn_stringbuf_create_ensure((apr_size_t)new_total_size, scratch_pool); - svn_stringbuf_t *compressed - = svn_stringbuf_create_empty(scratch_pool); - stream = svn_stream_from_stringbuf(uncompressed, scratch_pool); - - /* write the header*/ - SVN_ERR(serialize_revprops_header(stream, revprops->start_revision + start, - revprops->sizes, start, end, - scratch_pool)); + svn_packed__data_root_t *root = svn_packed__data_create_root(scratch_pool); + svn_packed__byte_stream_t *revprops_stream + = svn_packed__create_bytes_stream(root); /* append the serialized revprops */ for (i = start; i < end; ++i) - if (i == changed_index) - { - SVN_ERR(svn_stream_write(stream, - new_serialized->data, - &new_serialized->len)); - } - else - { - apr_size_t size - = (apr_size_t)APR_ARRAY_IDX(revprops->sizes, i, apr_off_t); - apr_size_t offset - = (apr_size_t)APR_ARRAY_IDX(revprops->offsets, i, apr_off_t); - - SVN_ERR(svn_stream_write(stream, - revprops->packed_revprops->data + offset, - &size)); - } - - /* flush the stream buffer (if any) to our underlying data buffer */ - SVN_ERR(svn_stream_close(stream)); + { + const svn_string_t *props + = &APR_ARRAY_IDX(revprops->revprops, i, svn_string_t); - /* compress / store the data */ - SVN_ERR(svn__compress(uncompressed, - compressed, - ffd->compress_packed_revprops - ? SVN_DELTA_COMPRESSION_LEVEL_DEFAULT - : SVN_DELTA_COMPRESSION_LEVEL_NONE)); + svn_packed__add_bytes(revprops_stream, props->data, props->len); + } - /* finally, write the content to the target stream and close it */ - SVN_ERR(svn_stream_write(file_stream, compressed->data, &compressed->len)); - SVN_ERR(svn_stream_close(file_stream)); + /* Write to file. */ + SVN_ERR(write_packed_data_checksummed(root, file, scratch_pool)); return SVN_NO_ERROR; } -/* Allocate a new pack file name for revisions - * [REVPROPS->START_REVISION + START, REVPROPS->START_REVISION + END - 1] - * of REVPROPS->MANIFEST. Add the name of old file to FILES_TO_DELETE, - * auto-create that array if necessary. Return an open file stream to - * the new file in *STREAM allocated in RESULT_POOL. Allocate the paths - * in *FILES_TO_DELETE from the same pool that contains the array itself. +/* Allocate a new pack file name for revisions starting at START_REV in + * REVPROPS->MANIFEST. Add the name of old file to FILES_TO_DELETE, + * auto-create that array if necessary. Return an open file *FILE that is + * allocated in RESULT_POOL. Allocate the paths in *FILES_TO_DELETE from + * the same pool that contains the array itself. Schedule necessary fsync + * calls in BATCH. * * Use SCRATCH_POOL for temporary allocations. */ static svn_error_t * -repack_stream_open(svn_stream_t **stream, - svn_fs_t *fs, - packed_revprops_t *revprops, - int start, - int end, - apr_array_header_t **files_to_delete, - apr_pool_t *result_pool, - apr_pool_t *scratch_pool) +repack_file_open(apr_file_t **file, + svn_fs_t *fs, + packed_revprops_t *revprops, + svn_revnum_t start_rev, + apr_array_header_t **files_to_delete, + svn_fs_x__batch_fsync_t *batch, + apr_pool_t *result_pool, + apr_pool_t *scratch_pool) { - apr_int64_t tag; - const char *tag_string; - svn_string_t *new_filename; - int i; - apr_file_t *file; - int manifest_offset - = (int)(revprops->start_revision - revprops->manifest_start); + manifest_entry_t new_entry; + const char *new_path; + int idx; - /* get the old (= current) file name and enlist it for later deletion */ - const char *old_filename = APR_ARRAY_IDX(revprops->manifest, - start + manifest_offset, - const char*); + /* We always replace whole pack files - possibly by more than one new file. + * When we create the file for the first part of the pack, enlist the old + * one for later deletion */ + SVN_ERR_ASSERT(start_rev >= revprops->entry.start_rev); if (*files_to_delete == NULL) *files_to_delete = apr_array_make(result_pool, 3, sizeof(const char*)); - APR_ARRAY_PUSH(*files_to_delete, const char*) - = svn_dirent_join(revprops->folder, old_filename, - (*files_to_delete)->pool); - - /* increase the tag part, i.e. the counter after the dot */ - tag_string = strchr(old_filename, '.'); - if (tag_string == NULL) - return svn_error_createf(SVN_ERR_FS_CORRUPT, NULL, - _("Packed file '%s' misses a tag"), - old_filename); + if (revprops->entry.start_rev == start_rev) + APR_ARRAY_PUSH(*files_to_delete, const char*) + = get_revprop_pack_filepath(revprops, &revprops->entry, + (*files_to_delete)->pool); - SVN_ERR(svn_cstring_atoi64(&tag, tag_string + 1)); - new_filename = svn_string_createf((*files_to_delete)->pool, - "%ld.%" APR_INT64_T_FMT, - revprops->start_revision + start, - ++tag); + /* Initialize the new manifest entry. Bump the tag part. */ + new_entry.start_rev = start_rev; + new_entry.tag = revprops->entry.tag + 1; /* update the manifest to point to the new file */ - for (i = start; i < end; ++i) - APR_ARRAY_IDX(revprops->manifest, i + manifest_offset, const char*) - = new_filename->data; + idx = get_entry(revprops->manifest, start_rev); + if (revprops->entry.start_rev == start_rev) + APR_ARRAY_IDX(revprops->manifest, idx, manifest_entry_t) = new_entry; + else + svn_sort__array_insert(revprops->manifest, &new_path, idx + 1); - /* create a file stream for the new file */ - SVN_ERR(svn_io_file_open(&file, svn_dirent_join(revprops->folder, - new_filename->data, - scratch_pool), - APR_WRITE | APR_CREATE, APR_OS_DEFAULT, - result_pool)); - *stream = svn_stream_from_aprfile2(file, FALSE, result_pool); + /* open the file */ + new_path = get_revprop_pack_filepath(revprops, &new_entry, scratch_pool); + SVN_ERR(svn_fs_x__batch_fsync_open_file(file, batch, new_path, + scratch_pool)); return SVN_NO_ERROR; } +/* Return the length of the serialized reprop list of index I in REVPROPS. */ +static apr_size_t +props_len(packed_revprops_t *revprops, + int i) +{ + return APR_ARRAY_IDX(revprops->revprops, i, svn_string_t).len; +} + /* For revision REV in filesystem FS, set the revision properties to * PROPLIST. Return a new file in *TMP_PATH that the caller shall move * to *FINAL_PATH to make the change visible. Files to be deleted will * be listed in *FILES_TO_DELETE which may remain unchanged / unallocated. + * Schedule necessary fsync calls in BATCH. * * Allocate output values in RESULT_POOL and temporaries from SCRATCH_POOL. */ @@ -1443,88 +1236,87 @@ write_packed_revprop(const char **final_path, svn_fs_t *fs, svn_revnum_t rev, apr_hash_t *proplist, + svn_fs_x__batch_fsync_t *batch, apr_pool_t *result_pool, apr_pool_t *scratch_pool) { svn_fs_x__data_t *ffd = fs->fsap_data; packed_revprops_t *revprops; - apr_int64_t generation = 0; svn_stream_t *stream; + apr_file_t *file; svn_stringbuf_t *serialized; - apr_off_t new_total_size; + apr_size_t new_total_size; int changed_index; + int count; /* read the current revprop generation. This value will not change * while we hold the global write lock to this FS. */ if (has_revprop_cache(fs, scratch_pool)) - SVN_ERR(read_revprop_generation(&generation, fs, scratch_pool)); + SVN_ERR(read_revprop_generation(fs, scratch_pool)); /* read contents of the current pack file */ - SVN_ERR(read_pack_revprop(&revprops, fs, rev, generation, TRUE, + SVN_ERR(read_pack_revprop(&revprops, fs, rev, TRUE, scratch_pool, scratch_pool)); /* serialize the new revprops */ serialized = svn_stringbuf_create_empty(scratch_pool); stream = svn_stream_from_stringbuf(serialized, scratch_pool); - SVN_ERR(svn_hash_write2(proplist, stream, SVN_HASH_TERMINATOR, - scratch_pool)); + SVN_ERR(svn_fs_x__write_properties(stream, proplist, scratch_pool)); SVN_ERR(svn_stream_close(stream)); - /* calculate the size of the new data */ - changed_index = (int)(rev - revprops->start_revision); + /* estimate the size of the new data */ + count = revprops->revprops->nelts; + changed_index = (int)(rev - revprops->entry.start_rev); new_total_size = revprops->total_size - revprops->serialized_size + serialized->len - + (revprops->offsets->nelts + 2) * SVN_INT64_BUFFER_SIZE; + + (count + 2) * SVN_INT64_BUFFER_SIZE; - APR_ARRAY_IDX(revprops->sizes, changed_index, apr_off_t) = serialized->len; + APR_ARRAY_IDX(revprops->revprops, changed_index, svn_string_t) + = *svn_stringbuf__morph_into_string(serialized); /* can we put the new data into the same pack as the before? */ - if ( new_total_size < ffd->revprop_pack_size - || revprops->sizes->nelts == 1) + if (new_total_size < ffd->revprop_pack_size || count == 1) { /* simply replace the old pack file with new content as we do it * in the non-packed case */ - *final_path = svn_dirent_join(revprops->folder, revprops->filename, - result_pool); - SVN_ERR(svn_stream_open_unique(&stream, tmp_path, revprops->folder, - svn_io_file_del_none, result_pool, - scratch_pool)); - SVN_ERR(repack_revprops(fs, revprops, 0, revprops->sizes->nelts, - changed_index, serialized, new_total_size, - stream, scratch_pool)); + *final_path = get_revprop_pack_filepath(revprops, &revprops->entry, + result_pool); + *tmp_path = apr_pstrcat(result_pool, *final_path, ".tmp", SVN_VA_NULL); + SVN_ERR(svn_fs_x__batch_fsync_open_file(&file, batch, *tmp_path, + scratch_pool)); + SVN_ERR(repack_revprops(fs, revprops, 0, count, + new_total_size, file, scratch_pool)); } else { /* split the pack file into two of roughly equal size */ - int right_count, left_count, i; + int right_count, left_count; int left = 0; - int right = revprops->sizes->nelts - 1; - apr_off_t left_size = 2 * SVN_INT64_BUFFER_SIZE; - apr_off_t right_size = 2 * SVN_INT64_BUFFER_SIZE; + int right = count - 1; + apr_size_t left_size = 2 * SVN_INT64_BUFFER_SIZE; + apr_size_t right_size = 2 * SVN_INT64_BUFFER_SIZE; /* let left and right side grow such that their size difference * is minimal after each step. */ while (left <= right) - if ( left_size + APR_ARRAY_IDX(revprops->sizes, left, apr_off_t) - < right_size + APR_ARRAY_IDX(revprops->sizes, right, apr_off_t)) + if ( left_size + props_len(revprops, left) + < right_size + props_len(revprops, right)) { - left_size += APR_ARRAY_IDX(revprops->sizes, left, apr_off_t) - + SVN_INT64_BUFFER_SIZE; + left_size += props_len(revprops, left) + SVN_INT64_BUFFER_SIZE; ++left; } else { - right_size += APR_ARRAY_IDX(revprops->sizes, right, apr_off_t) - + SVN_INT64_BUFFER_SIZE; + right_size += props_len(revprops, right) + SVN_INT64_BUFFER_SIZE; --right; } /* since the items need much less than SVN_INT64_BUFFER_SIZE * bytes to represent their length, the split may not be optimal */ left_count = left; - right_count = revprops->sizes->nelts - left; + right_count = count - left; /* if new_size is large, one side may exceed the pack size limit. * In that case, split before and after the modified revprop.*/ @@ -1532,7 +1324,7 @@ write_packed_revprop(const char **final_path, || right_size > ffd->revprop_pack_size) { left_count = changed_index; - right_count = revprops->sizes->nelts - left_count - 1; + right_count = count - left_count - 1; } /* Allocate this here such that we can call the repack functions with @@ -1544,54 +1336,40 @@ write_packed_revprop(const char **final_path, /* write the new, split files */ if (left_count) { - SVN_ERR(repack_stream_open(&stream, fs, revprops, 0, - left_count, files_to_delete, - scratch_pool, scratch_pool)); + SVN_ERR(repack_file_open(&file, fs, revprops, + revprops->entry.start_rev, + files_to_delete, batch, + scratch_pool, scratch_pool)); SVN_ERR(repack_revprops(fs, revprops, 0, left_count, - changed_index, serialized, new_total_size, - stream, scratch_pool)); + new_total_size, file, scratch_pool)); } - if (left_count + right_count < revprops->sizes->nelts) + if (left_count + right_count < count) { - SVN_ERR(repack_stream_open(&stream, fs, revprops, changed_index, - changed_index + 1, files_to_delete, - scratch_pool, scratch_pool)); + SVN_ERR(repack_file_open(&file, fs, revprops, rev, + files_to_delete, batch, + scratch_pool, scratch_pool)); SVN_ERR(repack_revprops(fs, revprops, changed_index, changed_index + 1, - changed_index, serialized, new_total_size, - stream, scratch_pool)); + new_total_size, file, scratch_pool)); } if (right_count) { - SVN_ERR(repack_stream_open(&stream, fs, revprops, - revprops->sizes->nelts - right_count, - revprops->sizes->nelts, - files_to_delete, scratch_pool, - scratch_pool)); - SVN_ERR(repack_revprops(fs, revprops, - revprops->sizes->nelts - right_count, - revprops->sizes->nelts, changed_index, - serialized, new_total_size, stream, - scratch_pool)); + SVN_ERR(repack_file_open(&file, fs, revprops, rev + 1, + files_to_delete, batch, + scratch_pool, scratch_pool)); + SVN_ERR(repack_revprops(fs, revprops, count - right_count, count, + new_total_size, file, scratch_pool)); } /* write the new manifest */ *final_path = svn_dirent_join(revprops->folder, PATH_MANIFEST, result_pool); - SVN_ERR(svn_stream_open_unique(&stream, tmp_path, revprops->folder, - svn_io_file_del_none, result_pool, - scratch_pool)); - - for (i = 0; i < revprops->manifest->nelts; ++i) - { - const char *filename = APR_ARRAY_IDX(revprops->manifest, i, - const char*); - SVN_ERR(svn_stream_printf(stream, scratch_pool, "%s\n", filename)); - } - - SVN_ERR(svn_stream_close(stream)); + *tmp_path = apr_pstrcat(result_pool, *final_path, ".tmp", SVN_VA_NULL); + SVN_ERR(svn_fs_x__batch_fsync_open_file(&file, batch, *tmp_path, + scratch_pool)); + SVN_ERR(write_manifest(file, revprops->manifest, scratch_pool)); } return SVN_NO_ERROR; @@ -1611,9 +1389,15 @@ svn_fs_x__set_revision_proplist(svn_fs_t *fs, const char *tmp_path; const char *perms_reference; apr_array_header_t *files_to_delete = NULL; + svn_fs_x__batch_fsync_t *batch; + svn_fs_x__data_t *ffd = fs->fsap_data; SVN_ERR(svn_fs_x__ensure_revision_exists(rev, fs, scratch_pool)); + /* Perform all fsyncs through this instance. */ + SVN_ERR(svn_fs_x__batch_fsync_create(&batch, ffd->flush_to_disk, + scratch_pool)); + /* this info will not change while we hold the global FS write lock */ is_packed = svn_fs_x__is_packed_revprop(fs, rev); @@ -1636,12 +1420,12 @@ svn_fs_x__set_revision_proplist(svn_fs_t *fs, /* Serialize the new revprop data */ if (is_packed) SVN_ERR(write_packed_revprop(&final_path, &tmp_path, &files_to_delete, - fs, rev, proplist, scratch_pool, + fs, rev, proplist, batch, scratch_pool, scratch_pool)); else SVN_ERR(write_non_packed_revprop(&final_path, &tmp_path, - fs, rev, proplist, scratch_pool, - scratch_pool)); + fs, rev, proplist, batch, + scratch_pool, scratch_pool)); /* We use the rev file of this revision as the perms reference, * because when setting revprops for the first time, the revprop @@ -1652,7 +1436,7 @@ svn_fs_x__set_revision_proplist(svn_fs_t *fs, /* Now, switch to the new revprop data. */ SVN_ERR(switch_to_new_revprop(fs, final_path, tmp_path, perms_reference, - files_to_delete, bump_generation, + files_to_delete, bump_generation, batch, scratch_pool)); return SVN_NO_ERROR; @@ -1668,20 +1452,14 @@ svn_fs_x__packed_revprop_available(svn_boolean_t *missing, svn_revnum_t revision, apr_pool_t *scratch_pool) { - svn_fs_x__data_t *ffd = fs->fsap_data; - svn_stringbuf_t *content = NULL; + svn_node_kind_t kind; + packed_revprops_t *revprops; + svn_error_t *err; /* try to read the manifest file */ - const char *folder = svn_fs_x__path_revprops_pack_shard(fs, revision, - scratch_pool); - const char *manifest_path = svn_dirent_join(folder, PATH_MANIFEST, - scratch_pool); - - svn_error_t *err = svn_fs_x__try_stringbuf_from_file(&content, - missing, - manifest_path, - FALSE, - scratch_pool); + revprops = apr_pcalloc(scratch_pool, sizeof(*revprops)); + revprops->revision = revision; + err = get_revprop_packname(fs, revprops, scratch_pool, scratch_pool); /* if the manifest cannot be read, consider the pack files inaccessible * even if the file itself exists. */ @@ -1691,114 +1469,93 @@ svn_fs_x__packed_revprop_available(svn_boolean_t *missing, return FALSE; } - if (*missing) - return FALSE; - - /* parse manifest content until we find the entry for REVISION. - * Revision 0 is never packed. */ - revision = revision < ffd->max_files_per_dir - ? revision - 1 - : revision % ffd->max_files_per_dir; - while (content->data) + /* the respective pack file must exist (and be a file) */ + err = svn_io_check_path(get_revprop_pack_filepath(revprops, + &revprops->entry, + scratch_pool), + &kind, scratch_pool); + if (err) { - char *next = strchr(content->data, '\n'); - if (next) - { - *next = 0; - ++next; - } - - if (revision-- == 0) - { - /* the respective pack file must exist (and be a file) */ - svn_node_kind_t kind; - err = svn_io_check_path(svn_dirent_join(folder, content->data, - scratch_pool), - &kind, scratch_pool); - if (err) - { - svn_error_clear(err); - return FALSE; - } - - *missing = kind == svn_node_none; - return kind == svn_node_file; - } - - content->data = next; + svn_error_clear(err); + return FALSE; } - return FALSE; + *missing = kind == svn_node_none; + return kind == svn_node_file; } /****** Packing FSX shards *********/ -svn_error_t * -svn_fs_x__copy_revprops(const char *pack_file_dir, - const char *pack_filename, - const char *shard_path, - svn_revnum_t start_rev, - svn_revnum_t end_rev, - apr_array_header_t *sizes, - apr_size_t total_size, - int compression_level, - svn_cancel_func_t cancel_func, - void *cancel_baton, - apr_pool_t *scratch_pool) +/* Copy revprop files for revisions [START_REV, END_REV) from SHARD_PATH + * in filesystem FS to the pack file at PACK_FILE_NAME in PACK_FILE_DIR. + * + * The file sizes have already been determined and written to SIZES. + * Please note that this function will be executed while the filesystem + * has been locked and that revprops files will therefore not be modified + * while the pack is in progress. + * + * COMPRESSION_LEVEL defines how well the resulting pack file shall be + * compressed or whether is shall be compressed at all. TOTAL_SIZE is + * a hint on which initial buffer size we should use to hold the pack file + * content. Schedule necessary fsync calls in BATCH. + * + * CANCEL_FUNC and CANCEL_BATON are used as usual. Temporary allocations + * are done in SCRATCH_POOL. + */ +static svn_error_t * +copy_revprops(svn_fs_t *fs, + const char *pack_file_dir, + const char *pack_filename, + const char *shard_path, + svn_revnum_t start_rev, + svn_revnum_t end_rev, + apr_array_header_t *sizes, + apr_size_t total_size, + int compression_level, + svn_fs_x__batch_fsync_t *batch, + svn_cancel_func_t cancel_func, + void *cancel_baton, + apr_pool_t *scratch_pool) { - svn_stream_t *pack_stream; apr_file_t *pack_file; svn_revnum_t rev; apr_pool_t *iterpool = svn_pool_create(scratch_pool); - svn_stream_t *stream; - - /* create empty data buffer and a write stream on top of it */ - svn_stringbuf_t *uncompressed - = svn_stringbuf_create_ensure(total_size, scratch_pool); - svn_stringbuf_t *compressed - = svn_stringbuf_create_empty(scratch_pool); - pack_stream = svn_stream_from_stringbuf(uncompressed, scratch_pool); - - /* write the pack file header */ - SVN_ERR(serialize_revprops_header(pack_stream, start_rev, sizes, 0, - sizes->nelts, iterpool)); - /* Some useful paths. */ - SVN_ERR(svn_io_file_open(&pack_file, svn_dirent_join(pack_file_dir, - pack_filename, - scratch_pool), - APR_WRITE | APR_CREATE, APR_OS_DEFAULT, - scratch_pool)); + svn_packed__data_root_t *root = svn_packed__data_create_root(scratch_pool); + svn_packed__byte_stream_t *stream + = svn_packed__create_bytes_stream(root); /* Iterate over the revisions in this shard, squashing them together. */ for (rev = start_rev; rev <= end_rev; rev++) { const char *path; + svn_stringbuf_t *props; svn_pool_clear(iterpool); /* Construct the file name. */ - path = svn_dirent_join(shard_path, apr_psprintf(iterpool, "%ld", rev), - iterpool); + path = svn_fs_x__path_revprops(fs, rev, iterpool); /* Copy all the bits from the non-packed revprop file to the end of * the pack file. */ - SVN_ERR(svn_stream_open_readonly(&stream, path, iterpool, iterpool)); - SVN_ERR(svn_stream_copy3(stream, pack_stream, - cancel_func, cancel_baton, iterpool)); + SVN_ERR(svn_stringbuf_from_file2(&props, path, iterpool)); + SVN_ERR_W(verify_checksum(props, iterpool), + apr_psprintf(iterpool, "Failed to read revprops for r%ld.", + rev)); + + svn_packed__add_bytes(stream, props->data, props->len); } - /* flush stream buffers to content buffer */ - SVN_ERR(svn_stream_close(pack_stream)); + /* Create the auto-fsync'ing pack file. */ + SVN_ERR(svn_fs_x__batch_fsync_open_file(&pack_file, batch, + svn_dirent_join(pack_file_dir, + pack_filename, + scratch_pool), + scratch_pool)); - /* compress the content (or just store it for COMPRESSION_LEVEL 0) */ - SVN_ERR(svn__compress(uncompressed, compressed, compression_level)); - - /* write the pack file content to disk */ - stream = svn_stream_from_aprfile2(pack_file, FALSE, scratch_pool); - SVN_ERR(svn_stream_write(stream, compressed->data, &compressed->len)); - SVN_ERR(svn_stream_close(stream)); + /* write all to disk */ + SVN_ERR(write_packed_data_checksummed(root, pack_file, scratch_pool)); svn_pool_destroy(iterpool); @@ -1806,49 +1563,62 @@ svn_fs_x__copy_revprops(const char *pack_file_dir, } svn_error_t * -svn_fs_x__pack_revprops_shard(const char *pack_file_dir, +svn_fs_x__pack_revprops_shard(svn_fs_t *fs, + const char *pack_file_dir, const char *shard_path, apr_int64_t shard, int max_files_per_dir, - apr_off_t max_pack_size, + apr_int64_t max_pack_size, int compression_level, + svn_fs_x__batch_fsync_t *batch, svn_cancel_func_t cancel_func, void *cancel_baton, apr_pool_t *scratch_pool) { const char *manifest_file_path, *pack_filename = NULL; - svn_stream_t *manifest_stream; + apr_file_t *manifest_file; svn_revnum_t start_rev, end_rev, rev; - apr_off_t total_size; + apr_size_t total_size; apr_pool_t *iterpool = svn_pool_create(scratch_pool); apr_array_header_t *sizes; + apr_array_header_t *manifest; + + /* Sanitize config file values. */ + apr_size_t max_size = (apr_size_t)MIN(MAX(max_pack_size, 1), + SVN_MAX_OBJECT_SIZE); /* Some useful paths. */ manifest_file_path = svn_dirent_join(pack_file_dir, PATH_MANIFEST, scratch_pool); - /* Remove any existing pack file for this shard, since it is incomplete. */ - SVN_ERR(svn_io_remove_dir2(pack_file_dir, TRUE, cancel_func, cancel_baton, - scratch_pool)); - - /* Create the new directory and manifest file stream. */ - SVN_ERR(svn_io_dir_make(pack_file_dir, APR_OS_DEFAULT, scratch_pool)); - SVN_ERR(svn_stream_open_writable(&manifest_stream, manifest_file_path, - scratch_pool, scratch_pool)); + /* Create the manifest file. */ + SVN_ERR(svn_fs_x__batch_fsync_open_file(&manifest_file, batch, + manifest_file_path, scratch_pool)); /* revisions to handle. Special case: revision 0 */ start_rev = (svn_revnum_t) (shard * max_files_per_dir); end_rev = (svn_revnum_t) ((shard + 1) * (max_files_per_dir) - 1); if (start_rev == 0) - ++start_rev; - /* Special special case: if max_files_per_dir is 1, then at this point - start_rev == 1 and end_rev == 0 (!). Fortunately, everything just - works. */ + { + /* Never pack revprops for r0, just copy it. */ + SVN_ERR(svn_io_copy_file(svn_fs_x__path_revprops(fs, 0, iterpool), + svn_dirent_join(pack_file_dir, "p0", + scratch_pool), + TRUE, + iterpool)); + + ++start_rev; + /* Special special case: if max_files_per_dir is 1, then at this point + start_rev == 1 and end_rev == 0 (!). Fortunately, everything just + works. */ + } /* initialize the revprop size info */ - sizes = apr_array_make(scratch_pool, max_files_per_dir, sizeof(apr_off_t)); + sizes = apr_array_make(scratch_pool, max_files_per_dir, sizeof(apr_size_t)); total_size = 2 * SVN_INT64_BUFFER_SIZE; + manifest = apr_array_make(scratch_pool, 4, sizeof(manifest_entry_t)); + /* Iterate over the revisions in this shard, determine their size and * squashing them together into pack files. */ for (rev = start_rev; rev <= end_rev; rev++) @@ -1859,20 +1629,22 @@ svn_fs_x__pack_revprops_shard(const char *pack_file_dir, svn_pool_clear(iterpool); /* Get the size of the file. */ - path = svn_dirent_join(shard_path, apr_psprintf(iterpool, "%ld", rev), - iterpool); + path = svn_fs_x__path_revprops(fs, rev, iterpool); SVN_ERR(svn_io_stat(&finfo, path, APR_FINFO_SIZE, iterpool)); - /* if we already have started a pack file and this revprop cannot be - * appended to it, write the previous pack file. */ - if (sizes->nelts != 0 && - total_size + SVN_INT64_BUFFER_SIZE + finfo.size > max_pack_size) + /* If we already have started a pack file and this revprop cannot be + * appended to it, write the previous pack file. Note this overflow + * check works because we enforced MAX_SIZE <= SVN_MAX_OBJECT_SIZE. */ + if (sizes->nelts != 0 + && ( finfo.size > max_size + || total_size > max_size + || SVN_INT64_BUFFER_SIZE + finfo.size > max_size - total_size)) { - SVN_ERR(svn_fs_x__copy_revprops(pack_file_dir, pack_filename, - shard_path, start_rev, rev-1, - sizes, (apr_size_t)total_size, - compression_level, cancel_func, - cancel_baton, iterpool)); + SVN_ERR(copy_revprops(fs, pack_file_dir, pack_filename, + shard_path, start_rev, rev-1, + sizes, (apr_size_t)total_size, + compression_level, batch, cancel_func, + cancel_baton, iterpool)); /* next pack file starts empty again */ apr_array_clear(sizes); @@ -1883,66 +1655,31 @@ svn_fs_x__pack_revprops_shard(const char *pack_file_dir, /* Update the manifest. Allocate a file name for the current pack * file if it is a new one */ if (sizes->nelts == 0) - pack_filename = apr_psprintf(scratch_pool, "%ld.0", rev); + { + manifest_entry_t *entry = apr_array_push(manifest); + entry->start_rev = rev; + entry->tag = 0; - SVN_ERR(svn_stream_printf(manifest_stream, iterpool, "%s\n", - pack_filename)); + pack_filename = apr_psprintf(scratch_pool, "%ld.0", rev); + } /* add to list of files to put into the current pack file */ - APR_ARRAY_PUSH(sizes, apr_off_t) = finfo.size; + APR_ARRAY_PUSH(sizes, apr_size_t) = finfo.size; total_size += SVN_INT64_BUFFER_SIZE + finfo.size; } /* write the last pack file */ if (sizes->nelts != 0) - SVN_ERR(svn_fs_x__copy_revprops(pack_file_dir, pack_filename, shard_path, - start_rev, rev-1, sizes, - (apr_size_t)total_size, compression_level, - cancel_func, cancel_baton, iterpool)); + SVN_ERR(copy_revprops(fs, pack_file_dir, pack_filename, shard_path, + start_rev, rev-1, sizes, + (apr_size_t)total_size, compression_level, + batch, cancel_func, cancel_baton, iterpool)); - /* flush the manifest file and update permissions */ - SVN_ERR(svn_stream_close(manifest_stream)); - SVN_ERR(svn_io_copy_perms(shard_path, pack_file_dir, iterpool)); + SVN_ERR(write_manifest(manifest_file, manifest, iterpool)); + /* flush all data to disk and update permissions */ + SVN_ERR(svn_io_copy_perms(shard_path, pack_file_dir, iterpool)); svn_pool_destroy(iterpool); return SVN_NO_ERROR; } - -svn_error_t * -svn_fs_x__delete_revprops_shard(const char *shard_path, - apr_int64_t shard, - int max_files_per_dir, - svn_cancel_func_t cancel_func, - void *cancel_baton, - apr_pool_t *scratch_pool) -{ - if (shard == 0) - { - apr_pool_t *iterpool = svn_pool_create(scratch_pool); - int i; - - /* delete all files except the one for revision 0 */ - for (i = 1; i < max_files_per_dir; ++i) - { - const char *path; - svn_pool_clear(iterpool); - - path = svn_dirent_join(shard_path, - apr_psprintf(iterpool, "%d", i), - iterpool); - if (cancel_func) - SVN_ERR((*cancel_func)(cancel_baton)); - - SVN_ERR(svn_io_remove_file2(path, TRUE, iterpool)); - } - - svn_pool_destroy(iterpool); - } - else - SVN_ERR(svn_io_remove_dir2(shard_path, TRUE, - cancel_func, cancel_baton, scratch_pool)); - - return SVN_NO_ERROR; -} - diff --git a/subversion/libsvn_fs_x/revprops.h b/subversion/libsvn_fs_x/revprops.h index c4827c42c222..cf6f63896f1d 100644 --- a/subversion/libsvn_fs_x/revprops.h +++ b/subversion/libsvn_fs_x/revprops.h @@ -20,11 +20,13 @@ * ==================================================================== */ -#ifndef SVN_LIBSVN_FS__REVPROPS_H -#define SVN_LIBSVN_FS__REVPROPS_H +#ifndef SVN_LIBSVN_FS_X_REVPROPS_H +#define SVN_LIBSVN_FS_X_REVPROPS_H #include "svn_fs.h" +#include "batch_fsync.h" + #ifdef __cplusplus extern "C" { #endif /* __cplusplus */ @@ -39,44 +41,26 @@ svn_error_t * svn_fs_x__reset_revprop_generation_file(svn_fs_t *fs, apr_pool_t *scratch_pool); -/* In the filesystem FS, pack all revprop shards up to min_unpacked_rev. - * - * NOTE: Keep the old non-packed shards around until after the format bump. - * Otherwise, re-running upgrade will drop the packed revprop shard but - * have no unpacked data anymore. Call upgrade_cleanup_pack_revprops after - * the bump. - * - * NOTIFY_FUNC and NOTIFY_BATON as well as CANCEL_FUNC and CANCEL_BATON are - * used in the usual way. Temporary allocations are done in SCRATCH_POOL. - */ -svn_error_t * -svn_fs_x__upgrade_pack_revprops(svn_fs_t *fs, - svn_fs_upgrade_notify_t notify_func, - void *notify_baton, - svn_cancel_func_t cancel_func, - void *cancel_baton, - apr_pool_t *scratch_pool); +/* Invalidate the cached revprop generation value in FS->FSAP_DATA. + * This enforces a re-read upon the next revprop read. */ +void +svn_fs_x__invalidate_revprop_generation(svn_fs_t *fs); -/* In the filesystem FS, remove all non-packed revprop shards up to - * min_unpacked_rev. Temporary allocations are done in SCRATCH_POOL. - * - * NOTIFY_FUNC and NOTIFY_BATON as well as CANCEL_FUNC and CANCEL_BATON are - * used in the usual way. Cancellation is supported in the sense that we - * will cleanly abort the operation. However, there will be remnant shards - * that must be removed manually. +/* Utility function serializing PROPLIST into FILE and adding the checksum. + * Use SCRATCH_POOL for temporary allocations. * - * See upgrade_pack_revprops for more info. + * Call this only when creating initial revprop file contents. + * For modifications use svn_fs_x__set_revision_proplist. */ svn_error_t * -svn_fs_x__upgrade_cleanup_pack_revprops(svn_fs_t *fs, - svn_fs_upgrade_notify_t notify_func, - void *notify_baton, - svn_cancel_func_t cancel_func, - void *cancel_baton, - apr_pool_t *scratch_pool); +svn_fs_x__write_non_packed_revprops(apr_file_t *file, + apr_hash_t *proplist, + apr_pool_t *scratch_pool); /* Read the revprops for revision REV in FS and return them in *PROPLIST_P. * If BYPASS_CACHE is set, don't consult the disks but always read from disk. + * If REFRESH is set, update the revprop generation info; otherwise access + * potentially outdated cache data directly. * * Allocate the *PROPLIST_P in RESULT_POOL and use SCRATCH_POOL for temporary * allocations. @@ -86,6 +70,7 @@ svn_fs_x__get_revision_proplist(apr_hash_t **proplist_p, svn_fs_t *fs, svn_revnum_t rev, svn_boolean_t bypass_cache, + svn_boolean_t refresh, apr_pool_t *result_pool, apr_pool_t *scratch_pool); @@ -111,37 +96,9 @@ svn_fs_x__packed_revprop_available(svn_boolean_t *missing, /****** Packing FSX shards *********/ -/* Copy revprop files for revisions [START_REV, END_REV) from SHARD_PATH - * to the pack file at PACK_FILE_NAME in PACK_FILE_DIR. - * - * The file sizes have already been determined and written to SIZES. - * Please note that this function will be executed while the filesystem - * has been locked and that revprops files will therefore not be modified - * while the pack is in progress. - * - * COMPRESSION_LEVEL defines how well the resulting pack file shall be - * compressed or whether is shall be compressed at all. TOTAL_SIZE is - * a hint on which initial buffer size we should use to hold the pack file - * content. - * - * CANCEL_FUNC and CANCEL_BATON are used as usual. Temporary allocations - * are done in SCRATCH_POOL. - */ -svn_error_t * -svn_fs_x__copy_revprops(const char *pack_file_dir, - const char *pack_filename, - const char *shard_path, - svn_revnum_t start_rev, - svn_revnum_t end_rev, - apr_array_header_t *sizes, - apr_size_t total_size, - int compression_level, - svn_cancel_func_t cancel_func, - void *cancel_baton, - apr_pool_t *scratch_pool); - /* For the revprop SHARD at SHARD_PATH with exactly MAX_FILES_PER_DIR - * revprop files in it, create a packed shared at PACK_FILE_DIR. + * revprop files in it, create a packed shared at PACK_FILE_DIR in + * filesystem FS. Schedule necessary fsync calls in BATCH. * * COMPRESSION_LEVEL defines how well the resulting pack file shall be * compressed or whether is shall be compressed at all. Individual pack @@ -152,33 +109,20 @@ svn_fs_x__copy_revprops(const char *pack_file_dir, * allocations are done in SCRATCH_POOL. */ svn_error_t * -svn_fs_x__pack_revprops_shard(const char *pack_file_dir, +svn_fs_x__pack_revprops_shard(svn_fs_t *fs, + const char *pack_file_dir, const char *shard_path, apr_int64_t shard, int max_files_per_dir, - apr_off_t max_pack_size, + apr_int64_t max_pack_size, int compression_level, + svn_fs_x__batch_fsync_t *batch, svn_cancel_func_t cancel_func, void *cancel_baton, apr_pool_t *scratch_pool); -/* Delete the non-packed revprop SHARD at SHARD_PATH with exactly - * MAX_FILES_PER_DIR revprop files in it. If this is shard 0, keep the - * revprop file for revision 0. - * - * CANCEL_FUNC and CANCEL_BATON are used in the usual way. Temporary - * allocations are done in SCRATCH_POOL. - */ -svn_error_t * -svn_fs_x__delete_revprops_shard(const char *shard_path, - apr_int64_t shard, - int max_files_per_dir, - svn_cancel_func_t cancel_func, - void *cancel_baton, - apr_pool_t *scratch_pool); - #ifdef __cplusplus } #endif /* __cplusplus */ -#endif /* SVN_LIBSVN_FS__REVPROPS_H */ +#endif /* SVN_LIBSVN_FS_X_REVPROPS_H */ diff --git a/subversion/libsvn_fs_x/string_table.c b/subversion/libsvn_fs_x/string_table.c index 7b3b6450cb52..e6a1cffdd2d9 100644 --- a/subversion/libsvn_fs_x/string_table.c +++ b/subversion/libsvn_fs_x/string_table.c @@ -376,7 +376,7 @@ svn_fs_x__string_table_builder_estimate_size(string_table_builder_t *builder) static void create_table(string_sub_table_t *target, builder_table_t *source, - apr_pool_t *pool, + apr_pool_t *result_pool, apr_pool_t *scratch_pool) { int i = 0; @@ -387,7 +387,8 @@ create_table(string_sub_table_t *target, /* pack sub-strings */ target->short_string_count = (apr_size_t)source->short_strings->nelts; - target->short_strings = apr_palloc(pool, sizeof(*target->short_strings) * + target->short_strings = apr_palloc(result_pool, + sizeof(*target->short_strings) * target->short_string_count); for (i = 0; i < source->short_strings->nelts; ++i) { @@ -433,13 +434,14 @@ create_table(string_sub_table_t *target, /* pack long strings */ target->long_string_count = (apr_size_t)source->long_strings->nelts; - target->long_strings = apr_palloc(pool, sizeof(*target->long_strings) * + target->long_strings = apr_palloc(result_pool, + sizeof(*target->long_strings) * target->long_string_count); for (i = 0; i < source->long_strings->nelts; ++i) { svn_string_t *string = &target->long_strings[i]; *string = APR_ARRAY_IDX(source->long_strings, i, svn_string_t); - string->data = apr_pstrmemdup(pool, string->data, string->len); + string->data = apr_pstrmemdup(result_pool, string->data, string->len); } data->len += PADDING; /* add a few extra bytes at the end of the buffer @@ -447,25 +449,25 @@ create_table(string_sub_table_t *target, assert(data->len < data->blocksize); memset(data->data + data->len - PADDING, 0, PADDING); - target->data = apr_pmemdup(pool, data->data, data->len); + target->data = apr_pmemdup(result_pool, data->data, data->len); target->data_size = data->len; } string_table_t * svn_fs_x__string_table_create(const string_table_builder_t *builder, - apr_pool_t *pool) + apr_pool_t *result_pool) { apr_size_t i; - string_table_t *result = apr_pcalloc(pool, sizeof(*result)); + string_table_t *result = apr_pcalloc(result_pool, sizeof(*result)); result->size = (apr_size_t)builder->tables->nelts; result->sub_tables - = apr_pcalloc(pool, result->size * sizeof(*result->sub_tables)); + = apr_pcalloc(result_pool, result->size * sizeof(*result->sub_tables)); for (i = 0; i < result->size; ++i) create_table(&result->sub_tables[i], APR_ARRAY_IDX(builder->tables, i, builder_table_t*), - pool, + result_pool, builder->pool); return result; @@ -542,7 +544,7 @@ const char* svn_fs_x__string_table_get(const string_table_t *table, apr_size_t idx, apr_size_t *length, - apr_pool_t *pool) + apr_pool_t *result_pool) { apr_size_t table_number = idx >> TABLE_SHIFT; apr_size_t sub_index = idx & STRING_INDEX_MASK; @@ -557,7 +559,7 @@ svn_fs_x__string_table_get(const string_table_t *table, if (length) *length = sub_table->long_strings[sub_index].len; - return apr_pstrmemdup(pool, + return apr_pstrmemdup(result_pool, sub_table->long_strings[sub_index].data, sub_table->long_strings[sub_index].len); } @@ -568,7 +570,7 @@ svn_fs_x__string_table_get(const string_table_t *table, { string_header_t *header = sub_table->short_strings + sub_index; apr_size_t len = header->head_length + header->tail_length; - char *result = apr_palloc(pool, len + PADDING); + char *result = apr_palloc(result_pool, len + PADDING); if (length) *length = len; @@ -579,7 +581,7 @@ svn_fs_x__string_table_get(const string_table_t *table, } } - return apr_pstrmemdup(pool, "", 0); + return apr_pstrmemdup(result_pool, "", 0); } svn_error_t * @@ -830,7 +832,7 @@ const char* svn_fs_x__string_table_get_func(const string_table_t *table, apr_size_t idx, apr_size_t *length, - apr_pool_t *pool) + apr_pool_t *result_pool) { apr_size_t table_number = idx >> TABLE_SHIFT; apr_size_t sub_index = idx & STRING_INDEX_MASK; @@ -861,7 +863,7 @@ svn_fs_x__string_table_get_func(const string_table_t *table, if (length) *length = long_strings[sub_index].len; - return apr_pstrmemdup(pool, + return apr_pstrmemdup(result_pool, str_data, long_strings[sub_index].len); } @@ -889,7 +891,7 @@ svn_fs_x__string_table_get_func(const string_table_t *table, /* reconstruct the char data and return it */ header = table_copy.short_strings + sub_index; len = header->head_length + header->tail_length; - result = apr_palloc(pool, len + PADDING); + result = apr_palloc(result_pool, len + PADDING); if (length) *length = len; diff --git a/subversion/libsvn_fs_x/string_table.h b/subversion/libsvn_fs_x/string_table.h index f7ab47698777..eed057232c9f 100644 --- a/subversion/libsvn_fs_x/string_table.h +++ b/subversion/libsvn_fs_x/string_table.h @@ -69,21 +69,21 @@ apr_size_t svn_fs_x__string_table_builder_estimate_size(string_table_builder_t *builder); /* From the given BUILDER object, create a string table object allocated - * in POOL that contains all strings previously added to BUILDER. + * in RESULT_POOL that contains all strings previously added to BUILDER. */ string_table_t * svn_fs_x__string_table_create(const string_table_builder_t *builder, - apr_pool_t *pool); + apr_pool_t *result_pool); /* Extract string number INDEX from TABLE and return a copy of it allocated - * in POOL. If LENGTH is not NULL, set *LENGTH to strlen() of the result - * string. Returns an empty string for invalid indexes. + * in RESULT_POOL. If LENGTH is not NULL, set *LENGTH to strlen() of the + * result string. Returns an empty string for invalid indexes. */ const char* svn_fs_x__string_table_get(const string_table_t *table, apr_size_t index, apr_size_t *length, - apr_pool_t *pool); + apr_pool_t *result_pool); /* Write a serialized representation of the string table TABLE to STREAM. * Use SCRATCH_POOL for temporary allocations. @@ -116,15 +116,15 @@ svn_fs_x__deserialize_string_table(void *buffer, string_table_t **table); /* Extract string number INDEX from the cache serialized representation at - * TABLE and return a copy of it allocated in POOL. If LENGTH is not NULL, - * set *LENGTH to strlen() of the result string. Returns an empty string - * for invalid indexes. + * TABLE and return a copy of it allocated in RESULT_POOL. If LENGTH is not + * NULL, set *LENGTH to strlen() of the result string. Returns an empty + * string for invalid indexes. */ const char* svn_fs_x__string_table_get_func(const string_table_t *table, apr_size_t idx, apr_size_t *length, - apr_pool_t *pool); + apr_pool_t *result_pool); #ifdef __cplusplus } diff --git a/subversion/libsvn_fs_x/temp_serializer.c b/subversion/libsvn_fs_x/temp_serializer.c index 65a2c3fdefde..efabf1b60771 100644 --- a/subversion/libsvn_fs_x/temp_serializer.c +++ b/subversion/libsvn_fs_x/temp_serializer.c @@ -71,13 +71,13 @@ encode_number(apr_int64_t number, char *key_buffer) const char* svn_fs_x__combine_number_and_string(apr_int64_t number, const char *string, - apr_pool_t *pool) + apr_pool_t *result_pool) { apr_size_t len = strlen(string); /* number part requires max. 10x7 bits + 1 space. * Add another 1 for the terminal 0 */ - char *key_buffer = apr_palloc(pool, len + 12); + char *key_buffer = apr_palloc(result_pool, len + 12); const char *key = key_buffer; /* Prepend the number to the string and separate them by space. No other @@ -120,7 +120,7 @@ serialize_svn_string(svn_temp_serializer__context_t *context, /* Utility function to deserialize the STRING inside the BUFFER. */ static void -deserialize_svn_string(void *buffer, svn_string_t **string) +deserialize_svn_string(const void *buffer, svn_string_t **string) { svn_temp_deserializer__resolve(buffer, (void **)string); if (*string == NULL) @@ -173,14 +173,14 @@ svn_fs_x__serialize_apr_array(svn_temp_serializer__context_t *context, void svn_fs_x__deserialize_apr_array(void *buffer, apr_array_header_t **array, - apr_pool_t *pool) + apr_pool_t *result_pool) { svn_temp_deserializer__resolve(buffer, (void **)array); if (*array == NULL) return; svn_temp_deserializer__resolve(*array, (void **)&(*array)->elts); - (*array)->pool = pool; + (*array)->pool = result_pool; } /* auxilliary structure representing the content of a directory array */ @@ -190,6 +190,10 @@ typedef struct dir_data_t * (it's int because the directory is an APR array) */ int count; + /** Current length of the in-txn in-disk representation of the directory. + * SVN_INVALID_FILESIZE if unknown (i.e. committed data). */ + svn_filesize_t txn_filesize; + /* number of unused dir entry buckets in the index */ apr_size_t over_provision; @@ -224,7 +228,7 @@ serialize_dir_entry(svn_temp_serializer__context_t *context, svn_temp_serializer__push(context, (const void * const *)entry_p, - sizeof(svn_fs_x__dirent_t)); + sizeof(**entry_p)); svn_temp_serializer__add_string(context, &entry->name); @@ -234,18 +238,19 @@ serialize_dir_entry(svn_temp_serializer__context_t *context, svn_temp_serializer__pop(context); } -/* Utility function to serialize the ENTRIES into a new serialization +/* Utility function to serialize the DIR into a new serialization * context to be returned. * * Temporary allocation will be made form SCRATCH_POOL. */ static svn_temp_serializer__context_t * -serialize_dir(apr_array_header_t *entries, +serialize_dir(svn_fs_x__dir_data_t *dir, apr_pool_t *scratch_pool) { dir_data_t dir_data; int i = 0; svn_temp_serializer__context_t *context; + apr_array_header_t *entries = dir->entries; /* calculate sizes */ int count = entries->nelts; @@ -254,8 +259,12 @@ serialize_dir(apr_array_header_t *entries, * sizeof(svn_fs_x__dirent_t*); apr_size_t lengths_len = (count + over_provision) * sizeof(apr_uint32_t); + /* Estimate the size of a directory entry + its name. */ + enum { ENTRY_SIZE = sizeof(svn_fs_x__dirent_t) + 32 }; + /* copy the hash entries to an auxiliary struct of known layout */ dir_data.count = count; + dir_data.txn_filesize = dir->txn_filesize; dir_data.over_provision = over_provision; dir_data.operations = 0; dir_data.entries = apr_palloc(scratch_pool, entries_len); @@ -268,7 +277,8 @@ serialize_dir(apr_array_header_t *entries, * estimate for the size of the buffer that we will need. */ context = svn_temp_serializer__init(&dir_data, sizeof(dir_data), - 50 + count * 200 + entries_len, + 50 + count * ENTRY_SIZE + + entries_len + lengths_len, scratch_pool); /* serialize entries references */ @@ -292,24 +302,32 @@ serialize_dir(apr_array_header_t *entries, return context; } -/* Utility function to reconstruct a dir entries array from serialized data - * in BUFFER and DIR_DATA. Allocation will be made form POOL. +/* Utility function to reconstruct a dir entries struct from serialized data + * in BUFFER and DIR_DATA. Allocation will be made form RESULT_POOL. */ -static apr_array_header_t * -deserialize_dir(void *buffer, dir_data_t *dir_data, apr_pool_t *pool) +static svn_fs_x__dir_data_t * +deserialize_dir(void *buffer, + dir_data_t *dir_data, + apr_pool_t *result_pool) { - apr_array_header_t *result - = apr_array_make(pool, dir_data->count, sizeof(svn_fs_x__dirent_t *)); + svn_fs_x__dir_data_t *result; apr_size_t i; apr_size_t count; svn_fs_x__dirent_t *entry; svn_fs_x__dirent_t **entries; + /* Construct empty directory object. */ + result = apr_pcalloc(result_pool, sizeof(*result)); + result->entries + = apr_array_make(result_pool, dir_data->count, + sizeof(svn_fs_x__dirent_t *)); + result->txn_filesize = dir_data->txn_filesize; + /* resolve the reference to the entries array */ svn_temp_deserializer__resolve(buffer, (void **)&dir_data->entries); entries = dir_data->entries; - /* fixup the references within each entry and add it to the hash */ + /* fixup the references within each entry and add it to the RESULT */ for (i = 0, count = dir_data->count; i < count; ++i) { svn_temp_deserializer__resolve(entries, (void **)&entries[i]); @@ -319,16 +337,19 @@ deserialize_dir(void *buffer, dir_data_t *dir_data, apr_pool_t *pool) svn_temp_deserializer__resolve(entry, (void **)&entry->name); /* add the entry to the hash */ - APR_ARRAY_PUSH(result, svn_fs_x__dirent_t *) = entry; + APR_ARRAY_PUSH(result->entries, svn_fs_x__dirent_t *) = entry; } /* return the now complete hash */ return result; } -void -svn_fs_x__noderev_serialize(svn_temp_serializer__context_t *context, - svn_fs_x__noderev_t * const *noderev_p) +/** + * Serialize a NODEREV_P within the serialization CONTEXT. + */ +static void +noderev_serialize(svn_temp_serializer__context_t *context, + svn_fs_x__noderev_t * const *noderev_p) { const svn_fs_x__noderev_t *noderev = *noderev_p; if (noderev == NULL) @@ -351,11 +372,12 @@ svn_fs_x__noderev_serialize(svn_temp_serializer__context_t *context, svn_temp_serializer__pop(context); } - -void -svn_fs_x__noderev_deserialize(void *buffer, - svn_fs_x__noderev_t **noderev_p, - apr_pool_t *pool) +/** + * Deserialize a NODEREV_P within the BUFFER and associate it with. + */ +static void +noderev_deserialize(void *buffer, + svn_fs_x__noderev_t **noderev_p) { svn_fs_x__noderev_t *noderev; @@ -451,7 +473,7 @@ svn_error_t * svn_fs_x__deserialize_txdelta_window(void **item, void *buffer, apr_size_t buffer_size, - apr_pool_t *pool) + apr_pool_t *result_pool) { svn_txdelta_window_t *window; @@ -474,38 +496,6 @@ svn_fs_x__deserialize_txdelta_window(void **item, return SVN_NO_ERROR; } -svn_error_t * -svn_fs_x__serialize_manifest(void **data, - apr_size_t *data_len, - void *in, - apr_pool_t *pool) -{ - apr_array_header_t *manifest = in; - - *data_len = sizeof(apr_off_t) *manifest->nelts; - *data = apr_palloc(pool, *data_len); - memcpy(*data, manifest->elts, *data_len); - - return SVN_NO_ERROR; -} - -svn_error_t * -svn_fs_x__deserialize_manifest(void **out, - void *data, - apr_size_t data_len, - apr_pool_t *pool) -{ - apr_array_header_t *manifest = apr_array_make(pool, 1, sizeof(apr_off_t)); - - manifest->nelts = (int) (data_len / sizeof(apr_off_t)); - manifest->nalloc = (int) (data_len / sizeof(apr_off_t)); - manifest->elts = (char*)data; - - *out = manifest; - - return SVN_NO_ERROR; -} - /* Auxiliary structure representing the content of a properties hash. This structure is much easier to (de-)serialize than an apr_hash. */ @@ -579,7 +569,7 @@ svn_fs_x__serialize_properties(void **data, /* create our auxiliary data structure */ properties.count = apr_hash_count(hash); properties.keys = apr_palloc(pool, sizeof(const char*) * (properties.count + 1)); - properties.values = apr_palloc(pool, sizeof(const char*) * properties.count); + properties.values = apr_palloc(pool, sizeof(const svn_string_t *) * properties.count); /* populate it with the hash entries */ for (hi = apr_hash_first(pool, hash), i=0; hi; hi = apr_hash_next(hi), ++i) @@ -611,9 +601,9 @@ svn_error_t * svn_fs_x__deserialize_properties(void **out, void *data, apr_size_t data_len, - apr_pool_t *pool) + apr_pool_t *result_pool) { - apr_hash_t *hash = svn_hash__make(pool); + apr_hash_t *hash = svn_hash__make(result_pool); properties_data_t *properties = (properties_data_t *)data; size_t i; @@ -661,7 +651,7 @@ svn_fs_x__serialize_node_revision(void **buffer, pool); /* serialize the noderev */ - svn_fs_x__noderev_serialize(context, &noderev); + noderev_serialize(context, &noderev); /* return serialized data */ serialized = svn_temp_serializer__get(context); @@ -675,13 +665,13 @@ svn_error_t * svn_fs_x__deserialize_node_revision(void **item, void *buffer, apr_size_t buffer_size, - apr_pool_t *pool) + apr_pool_t *result_pool) { /* Copy the _full_ buffer as it also contains the sub-structures. */ svn_fs_x__noderev_t *noderev = (svn_fs_x__noderev_t *)buffer; /* fixup of all pointers etc. */ - svn_fs_x__noderev_deserialize(noderev, &noderev, pool); + noderev_deserialize(noderev, &noderev); /* done */ *item = noderev; @@ -689,16 +679,18 @@ svn_fs_x__deserialize_node_revision(void **item, } /* Utility function that returns the directory serialized inside CONTEXT - * to DATA and DATA_LEN. */ + * to DATA and DATA_LEN. If OVERPROVISION is set, allocate some extra + * room for future in-place changes by svn_fs_x__replace_dir_entry. */ static svn_error_t * return_serialized_dir_context(svn_temp_serializer__context_t *context, void **data, - apr_size_t *data_len) + apr_size_t *data_len, + svn_boolean_t overprovision) { svn_stringbuf_t *serialized = svn_temp_serializer__get(context); *data = serialized->data; - *data_len = serialized->blocksize; + *data_len = overprovision ? serialized->blocksize : serialized->len; ((dir_data_t *)serialized->data)->len = serialized->len; return SVN_NO_ERROR; @@ -710,26 +702,27 @@ svn_fs_x__serialize_dir_entries(void **data, void *in, apr_pool_t *pool) { - apr_array_header_t *dir = in; + svn_fs_x__dir_data_t *dir = in; /* serialize the dir content into a new serialization context * and return the serialized data */ return return_serialized_dir_context(serialize_dir(dir, pool), data, - data_len); + data_len, + FALSE); } svn_error_t * svn_fs_x__deserialize_dir_entries(void **out, void *data, apr_size_t data_len, - apr_pool_t *pool) + apr_pool_t *result_pool) { /* Copy the _full_ buffer as it also contains the sub-structures. */ dir_data_t *dir_data = (dir_data_t *)data; /* reconstruct the hash from the serialized data */ - *out = deserialize_dir(dir_data, dir_data, pool); + *out = deserialize_dir(dir_data, dir_data, result_pool); return SVN_NO_ERROR; } @@ -749,6 +742,20 @@ svn_fs_x__get_sharded_offset(void **out, return SVN_NO_ERROR; } +svn_error_t * +svn_fs_x__extract_dir_filesize(void **out, + const void *data, + apr_size_t data_len, + void *baton, + apr_pool_t *pool) +{ + const dir_data_t *dir_data = data; + + *(svn_filesize_t *)out = dir_data->txn_filesize; + + return SVN_NO_ERROR; +} + /* Utility function that returns the lowest index of the first entry in * *ENTRIES that points to a dir entry with a name equal or larger than NAME. * If an exact match has been found, *FOUND will be set to TRUE. COUNT is @@ -832,6 +839,10 @@ svn_fs_x__extract_dir_entry(void **out, const apr_uint32_t *lengths = svn_temp_deserializer__ptr(data, (const void *const *)&dir_data->lengths); + /* Before we return, make sure we tell the caller this data is even still + relevant. */ + b->out_of_date = dir_data->txn_filesize != b->txn_filesize; + /* Special case: Early out for empty directories. That simplifies tests further down the road. */ *out = NULL; @@ -858,8 +869,9 @@ svn_fs_x__extract_dir_entry(void **out, if (found) b->hint = pos; - /* de-serialize that entry or return NULL, if no match has been found */ - if (found) + /* de-serialize that entry or return NULL, if no match has been found. + * Be sure to check that the directory contents is still up-to-date. */ + if (found && !b->out_of_date) { const svn_fs_x__dirent_t *source = svn_temp_deserializer__ptr(entries, (const void *const *)&entries[pos]); @@ -872,8 +884,7 @@ svn_fs_x__extract_dir_entry(void **out, apr_size_t size = lengths[pos]; /* copy & deserialize the entry */ - svn_fs_x__dirent_t *new_entry = apr_palloc(pool, size); - memcpy(new_entry, source, size); + svn_fs_x__dirent_t *new_entry = apr_pmemdup(pool, source, size); svn_temp_deserializer__resolve(new_entry, (void **)&new_entry->name); *(svn_fs_x__dirent_t **)out = new_entry; @@ -893,32 +904,34 @@ slowly_replace_dir_entry(void **data, { replace_baton_t *replace_baton = (replace_baton_t *)baton; dir_data_t *dir_data = (dir_data_t *)*data; - apr_array_header_t *dir; + svn_fs_x__dir_data_t *dir; int idx = -1; svn_fs_x__dirent_t *entry; + apr_array_header_t *entries; SVN_ERR(svn_fs_x__deserialize_dir_entries((void **)&dir, *data, dir_data->len, pool)); - entry = svn_fs_x__find_dir_entry(dir, replace_baton->name, &idx); + entries = dir->entries; + entry = svn_fs_x__find_dir_entry(entries, replace_baton->name, &idx); /* Replacement or removal? */ if (replace_baton->new_entry) { /* Replace ENTRY with / insert the NEW_ENTRY */ if (entry) - APR_ARRAY_IDX(dir, idx, svn_fs_x__dirent_t *) + APR_ARRAY_IDX(entries, idx, svn_fs_x__dirent_t *) = replace_baton->new_entry; else - svn_sort__array_insert(dir, &replace_baton->new_entry, idx); + svn_sort__array_insert(entries, &replace_baton->new_entry, idx); } else { /* Remove the old ENTRY. */ if (entry) - svn_sort__array_delete(dir, idx, 1); + svn_sort__array_delete(entries, idx, 1); } return svn_fs_x__serialize_dir_entries(data, data_len, dir, pool); @@ -940,6 +953,12 @@ svn_fs_x__replace_dir_entry(void **data, svn_temp_serializer__context_t *context; + /* update the cached file length info. + * Because we are writing to the cache, it is fair to assume that the + * caller made sure that the current contents is consistent with the + * previous state of the directory file. */ + dir_data->txn_filesize = replace_baton->txn_filesize; + /* after quite a number of operations, let's re-pack everything. * This is to limit the number of wasted space as we cannot overwrite * existing data but must always append. */ @@ -1012,9 +1031,7 @@ svn_fs_x__replace_dir_entry(void **data, serialize_dir_entry(context, &entries[pos], &length); /* return the updated serialized data */ - SVN_ERR (return_serialized_dir_context(context, - data, - data_len)); + SVN_ERR(return_serialized_dir_context(context, data, data_len, TRUE)); /* since the previous call may have re-allocated the buffer, the lengths * pointer may no longer point to the entry in that buffer. Therefore, @@ -1029,17 +1046,26 @@ svn_fs_x__replace_dir_entry(void **data, return SVN_NO_ERROR; } +svn_error_t * +svn_fs_x__reset_txn_filesize(void **data, + apr_size_t *data_len, + void *baton, + apr_pool_t *pool) +{ + dir_data_t *dir_data = (dir_data_t *)*data; + dir_data->txn_filesize = SVN_INVALID_FILESIZE; + + return SVN_NO_ERROR; +} + svn_error_t * svn_fs_x__serialize_rep_header(void **data, apr_size_t *data_len, void *in, apr_pool_t *pool) { - svn_fs_x__rep_header_t *copy = apr_palloc(pool, sizeof(*copy)); - *copy = *(svn_fs_x__rep_header_t *)in; - *data_len = sizeof(svn_fs_x__rep_header_t); - *data = copy; + *data = in; return SVN_NO_ERROR; } @@ -1048,12 +1074,8 @@ svn_error_t * svn_fs_x__deserialize_rep_header(void **out, void *data, apr_size_t data_len, - apr_pool_t *pool) + apr_pool_t *result_pool) { - svn_fs_x__rep_header_t *copy = apr_palloc(pool, sizeof(*copy)); - SVN_ERR_ASSERT(data_len == sizeof(*copy)); - - *copy = *(svn_fs_x__rep_header_t *)data; *out = data; return SVN_NO_ERROR; @@ -1088,8 +1110,7 @@ serialize_change(svn_temp_serializer__context_t *context, */ static void deserialize_change(void *buffer, - svn_fs_x__change_t **change_p, - apr_pool_t *pool) + svn_fs_x__change_t **change_p) { svn_fs_x__change_t * change; @@ -1105,47 +1126,29 @@ deserialize_change(void *buffer, svn_temp_deserializer__resolve(change, (void **)&change->copyfrom_path); } -/* Auxiliary structure representing the content of a svn_fs_x__change_t array. - This structure is much easier to (de-)serialize than an APR array. - */ -typedef struct changes_data_t -{ - /* number of entries in the array */ - int count; - - /* reference to the changes */ - svn_fs_x__change_t **changes; -} changes_data_t; - svn_error_t * svn_fs_x__serialize_changes(void **data, apr_size_t *data_len, void *in, apr_pool_t *pool) { - apr_array_header_t *array = in; - changes_data_t changes; + svn_fs_x__changes_list_t *changes = in; svn_temp_serializer__context_t *context; svn_stringbuf_t *serialized; int i; - /* initialize our auxiliary data structure and link it to the - * array elements */ - changes.count = array->nelts; - changes.changes = (svn_fs_x__change_t **)array->elts; - /* serialize it and all its elements */ - context = svn_temp_serializer__init(&changes, - sizeof(changes), - changes.count * 250, + context = svn_temp_serializer__init(changes, + sizeof(*changes), + changes->count * 250, pool); svn_temp_serializer__push(context, - (const void * const *)&changes.changes, - changes.count * sizeof(svn_fs_x__change_t*)); + (const void * const *)&changes->changes, + changes->count * sizeof(*changes->changes)); - for (i = 0; i < changes.count; ++i) - serialize_change(context, &changes.changes[i]); + for (i = 0; i < changes->count; ++i) + serialize_change(context, &changes->changes[i]); svn_temp_serializer__pop(context); @@ -1162,12 +1165,10 @@ svn_error_t * svn_fs_x__deserialize_changes(void **out, void *data, apr_size_t data_len, - apr_pool_t *pool) + apr_pool_t *result_pool) { int i; - changes_data_t *changes = (changes_data_t *)data; - apr_array_header_t *array = apr_array_make(pool, 0, - sizeof(svn_fs_x__change_t *)); + svn_fs_x__changes_list_t *changes = (svn_fs_x__changes_list_t *)data; /* de-serialize our auxiliary data structure */ svn_temp_deserializer__resolve(changes, (void**)&changes->changes); @@ -1175,163 +1176,10 @@ svn_fs_x__deserialize_changes(void **out, /* de-serialize each entry and add it to the array */ for (i = 0; i < changes->count; ++i) deserialize_change(changes->changes, - (svn_fs_x__change_t **)&changes->changes[i], - pool); - - /* Use the changes buffer as the array's data buffer - * (DATA remains valid for at least as long as POOL). */ - array->elts = (char *)changes->changes; - array->nelts = changes->count; - array->nalloc = changes->count; + (svn_fs_x__change_t **)&changes->changes[i]); /* done */ - *out = array; + *out = changes; return SVN_NO_ERROR; } - -/* Auxiliary structure representing the content of a svn_mergeinfo_t hash. - This structure is much easier to (de-)serialize than an APR array. - */ -typedef struct mergeinfo_data_t -{ - /* number of paths in the hash */ - unsigned count; - - /* COUNT keys (paths) */ - const char **keys; - - /* COUNT keys lengths (strlen of path) */ - apr_ssize_t *key_lengths; - - /* COUNT entries, each giving the number of ranges for the key */ - int *range_counts; - - /* all ranges in a single, concatenated buffer */ - svn_merge_range_t *ranges; -} mergeinfo_data_t; - -svn_error_t * -svn_fs_x__serialize_mergeinfo(void **data, - apr_size_t *data_len, - void *in, - apr_pool_t *pool) -{ - svn_mergeinfo_t mergeinfo = in; - mergeinfo_data_t merges; - svn_temp_serializer__context_t *context; - svn_stringbuf_t *serialized; - apr_hash_index_t *hi; - unsigned i; - int k; - apr_size_t range_count; - - /* initialize our auxiliary data structure */ - merges.count = apr_hash_count(mergeinfo); - merges.keys = apr_palloc(pool, sizeof(*merges.keys) * merges.count); - merges.key_lengths = apr_palloc(pool, sizeof(*merges.key_lengths) * - merges.count); - merges.range_counts = apr_palloc(pool, sizeof(*merges.range_counts) * - merges.count); - - i = 0; - range_count = 0; - for (hi = apr_hash_first(pool, mergeinfo); hi; hi = apr_hash_next(hi), ++i) - { - svn_rangelist_t *ranges; - apr_hash_this(hi, (const void**)&merges.keys[i], - &merges.key_lengths[i], - (void **)&ranges); - merges.range_counts[i] = ranges->nelts; - range_count += ranges->nelts; - } - - merges.ranges = apr_palloc(pool, sizeof(*merges.ranges) * range_count); - - i = 0; - for (hi = apr_hash_first(pool, mergeinfo); hi; hi = apr_hash_next(hi)) - { - svn_rangelist_t *ranges = apr_hash_this_val(hi); - for (k = 0; k < ranges->nelts; ++k, ++i) - merges.ranges[i] = *APR_ARRAY_IDX(ranges, k, svn_merge_range_t*); - } - - /* serialize it and all its elements */ - context = svn_temp_serializer__init(&merges, - sizeof(merges), - range_count * 30, - pool); - - /* keys array */ - svn_temp_serializer__push(context, - (const void * const *)&merges.keys, - merges.count * sizeof(*merges.keys)); - - for (i = 0; i < merges.count; ++i) - svn_temp_serializer__add_string(context, &merges.keys[i]); - - svn_temp_serializer__pop(context); - - /* key lengths array */ - svn_temp_serializer__add_leaf(context, - (const void * const *)&merges.key_lengths, - merges.count * sizeof(*merges.key_lengths)); - - /* range counts array */ - svn_temp_serializer__add_leaf(context, - (const void * const *)&merges.range_counts, - merges.count * sizeof(*merges.range_counts)); - - /* ranges */ - svn_temp_serializer__add_leaf(context, - (const void * const *)&merges.ranges, - range_count * sizeof(*merges.ranges)); - - /* return the serialized result */ - serialized = svn_temp_serializer__get(context); - - *data = serialized->data; - *data_len = serialized->len; - - return SVN_NO_ERROR; -} - -svn_error_t * -svn_fs_x__deserialize_mergeinfo(void **out, - void *data, - apr_size_t data_len, - apr_pool_t *pool) -{ - unsigned i; - int k, n; - mergeinfo_data_t *merges = (mergeinfo_data_t *)data; - svn_mergeinfo_t mergeinfo; - - /* de-serialize our auxiliary data structure */ - svn_temp_deserializer__resolve(merges, (void**)&merges->keys); - svn_temp_deserializer__resolve(merges, (void**)&merges->key_lengths); - svn_temp_deserializer__resolve(merges, (void**)&merges->range_counts); - svn_temp_deserializer__resolve(merges, (void**)&merges->ranges); - - /* de-serialize keys and add entries to the result */ - n = 0; - mergeinfo = svn_hash__make(pool); - for (i = 0; i < merges->count; ++i) - { - svn_rangelist_t *ranges = apr_array_make(pool, - merges->range_counts[i], - sizeof(svn_merge_range_t*)); - for (k = 0; k < merges->range_counts[i]; ++k, ++n) - APR_ARRAY_PUSH(ranges, svn_merge_range_t*) = &merges->ranges[n]; - - svn_temp_deserializer__resolve(merges->keys, - (void**)&merges->keys[i]); - apr_hash_set(mergeinfo, merges->keys[i], merges->key_lengths[i], ranges); - } - - /* done */ - *out = mergeinfo; - - return SVN_NO_ERROR; -} - diff --git a/subversion/libsvn_fs_x/temp_serializer.h b/subversion/libsvn_fs_x/temp_serializer.h index 80f500484d03..d6156d644743 100644 --- a/subversion/libsvn_fs_x/temp_serializer.h +++ b/subversion/libsvn_fs_x/temp_serializer.h @@ -20,8 +20,8 @@ * ==================================================================== */ -#ifndef SVN_LIBSVN_FS__TEMP_SERIALIZER_H -#define SVN_LIBSVN_FS__TEMP_SERIALIZER_H +#ifndef SVN_LIBSVN_FS_X_TEMP_SERIALIZER_H +#define SVN_LIBSVN_FS_X_TEMP_SERIALIZER_H #include "private/svn_temp_serializer.h" #include "fs.h" @@ -29,28 +29,12 @@ /** * Prepend the @a number to the @a string in a space efficient way such that * no other (number,string) combination can produce the same result. - * Allocate temporaries as well as the result from @a pool. + * Allocate the result from @a result_pool. */ const char* svn_fs_x__combine_number_and_string(apr_int64_t number, const char *string, - apr_pool_t *pool); - -/** - * Serialize a @a noderev_p within the serialization @a context. - */ -void -svn_fs_x__noderev_serialize(struct svn_temp_serializer__context_t *context, - svn_fs_x__noderev_t * const *noderev_p); - -/** - * Deserialize a @a noderev_p within the @a buffer and associate it with - * @a pool. - */ -void -svn_fs_x__noderev_deserialize(void *buffer, - svn_fs_x__noderev_t **noderev_p, - apr_pool_t *pool); + apr_pool_t *result_pool); /** * Serialize APR array @a *a within the serialization @a context. @@ -62,12 +46,12 @@ svn_fs_x__serialize_apr_array(struct svn_temp_serializer__context_t *context, /** * Deserialize APR @a *array within the @a buffer. Set its pool member to - * @a pool. The elements within the array must not contain pointers. + * @a result_pool. The elements within the array must not contain pointers. */ void svn_fs_x__deserialize_apr_array(void *buffer, apr_array_header_t **array, - apr_pool_t *pool); + apr_pool_t *result_pool); /** @@ -104,27 +88,7 @@ svn_error_t * svn_fs_x__deserialize_txdelta_window(void **item, void *buffer, apr_size_t buffer_size, - apr_pool_t *pool); - -/** - * Implements #svn_cache__serialize_func_t for a manifest - * (@a in is an #apr_array_header_t of apr_off_t elements). - */ -svn_error_t * -svn_fs_x__serialize_manifest(void **data, - apr_size_t *data_len, - void *in, - apr_pool_t *pool); - -/** - * Implements #svn_cache__deserialize_func_t for a manifest - * (@a *out is an #apr_array_header_t of apr_off_t elements). - */ -svn_error_t * -svn_fs_x__deserialize_manifest(void **out, - void *data, - apr_size_t data_len, - apr_pool_t *pool); + apr_pool_t *result_pool); /** * Implements #svn_cache__serialize_func_t for a properties hash @@ -144,7 +108,7 @@ svn_error_t * svn_fs_x__deserialize_properties(void **out, void *data, apr_size_t data_len, - apr_pool_t *pool); + apr_pool_t *result_pool); /** * Implements #svn_cache__serialize_func_t for #svn_fs_x__noderev_t @@ -162,10 +126,10 @@ svn_error_t * svn_fs_x__deserialize_node_revision(void **item, void *buffer, apr_size_t buffer_size, - apr_pool_t *pool); + apr_pool_t *result_pool); /** - * Implements #svn_cache__serialize_func_t for a directory contents array + * Implements #svn_cache__serialize_func_t for a #svn_fs_x__dir_data_t */ svn_error_t * svn_fs_x__serialize_dir_entries(void **data, @@ -174,13 +138,13 @@ svn_fs_x__serialize_dir_entries(void **data, apr_pool_t *pool); /** - * Implements #svn_cache__deserialize_func_t for a directory contents array + * Implements #svn_cache__deserialize_func_t for a #svn_fs_x__dir_data_t */ svn_error_t * svn_fs_x__deserialize_dir_entries(void **out, void *data, apr_size_t data_len, - apr_pool_t *pool); + apr_pool_t *result_pool); /** * Implements #svn_cache__partial_getter_func_t. Set (apr_off_t) @a *out @@ -194,6 +158,18 @@ svn_fs_x__get_sharded_offset(void **out, apr_pool_t *pool); /** + * Implements #svn_cache__partial_getter_func_t. + * Set (svn_filesize_t) @a *out to the filesize info stored with the + * serialized directory in @a data of @a data_len. @a baton is unused. + */ +svn_error_t * +svn_fs_x__extract_dir_filesize(void **out, + const void *data, + apr_size_t data_len, + void *baton, + apr_pool_t *pool); + +/** * Baton type to be used with svn_fs_x__extract_dir_entry. */ typedef struct svn_fs_x__ede_baton_t { @@ -202,12 +178,24 @@ typedef struct svn_fs_x__ede_baton_t /* Lookup hint [in / out] */ apr_size_t hint; + + /** Current length of the in-txn in-disk representation of the directory. + * SVN_INVALID_FILESIZE if unknown. */ + svn_filesize_t txn_filesize; + + /** Will be set by the callback. If FALSE, the cached data is out of date. + * We need this indicator because the svn_cache__t interface will always + * report the lookup as a success (FOUND==TRUE) if the generic lookup was + * successful -- regardless of what the entry extraction callback does. */ + svn_boolean_t out_of_date; } svn_fs_x__ede_baton_t; /** * Implements #svn_cache__partial_getter_func_t for a single * #svn_fs_x__dirent_t within a serialized directory contents hash, - * identified by its name (given in @a svn_fs_x__ede_baton_t @a *baton). + * identified by its name (in (svn_fs_x__ede_baton_t *) @a *baton). + * If the filesize specified in the baton does not match the cached + * value for this directory, @a *out will be NULL as well. */ svn_error_t * svn_fs_x__extract_dir_entry(void **out, @@ -220,7 +208,10 @@ svn_fs_x__extract_dir_entry(void **out, * Describes the change to be done to a directory: Set the entry * identify by @a name to the value @a new_entry. If the latter is * @c NULL, the entry shall be removed if it exists. Otherwise it - * will be replaced or automatically added, respectively. + * will be replaced or automatically added, respectively. The + * @a filesize allows readers to identify stale cache data (e.g. + * due to concurrent access to txns); writers use it to update the + * cached file size info. */ typedef struct replace_baton_t { @@ -229,6 +220,10 @@ typedef struct replace_baton_t /** directory entry to insert instead */ svn_fs_x__dirent_t *new_entry; + + /** Current length of the in-txn in-disk representation of the directory. + * SVN_INVALID_FILESIZE if unknown. */ + svn_filesize_t txn_filesize; } replace_baton_t; /** @@ -243,6 +238,17 @@ svn_fs_x__replace_dir_entry(void **data, apr_pool_t *pool); /** + * Implements #svn_cache__partial_setter_func_t for a #svn_fs_x__dir_data_t + * at @a *data, resetting its txn_filesize field to SVN_INVALID_FILESIZE. + * &a baton should be NULL. + */ +svn_error_t * +svn_fs_x__reset_txn_filesize(void **data, + apr_size_t *data_len, + void *baton, + apr_pool_t *pool); + +/** * Implements #svn_cache__serialize_func_t for a #svn_fs_x__rep_header_t. */ svn_error_t * @@ -258,11 +264,36 @@ svn_error_t * svn_fs_x__deserialize_rep_header(void **out, void *data, apr_size_t data_len, - apr_pool_t *pool); + apr_pool_t *result_pool); + +/*** Block of changes in a changed paths list. */ +typedef struct svn_fs_x__changes_list_t +{ + /* Offset of the first element in CHANGES within the changed paths list + on disk. */ + apr_off_t start_offset; + + /* Offset of the first element behind CHANGES within the changed paths + list on disk. */ + apr_off_t end_offset; + + /* End of list reached? This may have false negatives in case the number + of elements in the list is a multiple of our block / range size. */ + svn_boolean_t eol; + + /* Array of #svn_fs_x__change_t * representing a consecutive sub-range of + elements in a changed paths list. */ + + /* number of entries in the array */ + int count; + + /* reference to the changes */ + svn_fs_x__change_t **changes; + +} svn_fs_x__changes_list_t; /** - * Implements #svn_cache__serialize_func_t for an #apr_array_header_t of - * #svn_fs_x__change_t *. + * Implements #svn_cache__serialize_func_t for a #svn_fs_x__changes_list_t. */ svn_error_t * svn_fs_x__serialize_changes(void **data, @@ -271,31 +302,12 @@ svn_fs_x__serialize_changes(void **data, apr_pool_t *pool); /** - * Implements #svn_cache__deserialize_func_t for an #apr_array_header_t of - * #svn_fs_x__change_t *. + * Implements #svn_cache__deserialize_func_t for a #svn_fs_x__changes_list_t. */ svn_error_t * svn_fs_x__deserialize_changes(void **out, void *data, apr_size_t data_len, - apr_pool_t *pool); - -/** - * Implements #svn_cache__serialize_func_t for #svn_mergeinfo_t objects. - */ -svn_error_t * -svn_fs_x__serialize_mergeinfo(void **data, - apr_size_t *data_len, - void *in, - apr_pool_t *pool); - -/** - * Implements #svn_cache__deserialize_func_t for #svn_mergeinfo_t objects. - */ -svn_error_t * -svn_fs_x__deserialize_mergeinfo(void **out, - void *data, - apr_size_t data_len, - apr_pool_t *pool); + apr_pool_t *result_pool); #endif diff --git a/subversion/libsvn_fs_x/transaction.c b/subversion/libsvn_fs_x/transaction.c index 5f3adc595e03..52c958f3ece1 100644 --- a/subversion/libsvn_fs_x/transaction.c +++ b/subversion/libsvn_fs_x/transaction.c @@ -25,6 +25,7 @@ #include <assert.h> #include <apr_sha1.h> +#include "svn_error_codes.h" #include "svn_hash.h" #include "svn_props.h" #include "svn_sorts.h" @@ -41,6 +42,8 @@ #include "lock.h" #include "rep-cache.h" #include "index.h" +#include "batch_fsync.h" +#include "revprops.h" #include "private/svn_fs_util.h" #include "private/svn_fspath.h" @@ -279,17 +282,33 @@ with_some_lock_file(with_lock_baton_t *baton) ffd->has_write_lock = TRUE; } - /* nobody else will modify the repo state - => read HEAD & pack info once */ if (baton->is_inner_most_lock) { - err = svn_fs_x__update_min_unpacked_rev(fs, pool); + /* Use a separate sub-pool for the actual function body and a few + * file accesses. So, the lock-pool only contains the file locks. + */ + apr_pool_t *subpool = svn_pool_create(pool); + + /* nobody else will modify the repo state + => read HEAD & pack info once */ + err = svn_fs_x__update_min_unpacked_rev(fs, subpool); if (!err) - err = svn_fs_x__youngest_rev(&ffd->youngest_rev_cache, fs, pool); - } + err = svn_fs_x__youngest_rev(&ffd->youngest_rev_cache, fs, + subpool); + + /* We performed a few file operations. Clean the pool. */ + svn_pool_clear(subpool); - if (!err) - err = baton->body(baton->baton, pool); + if (!err) + err = baton->body(baton->baton, subpool); + + svn_pool_destroy(subpool); + } + else + { + /* Nested lock level */ + err = baton->body(baton->baton, pool); + } } if (baton->is_outer_most_lock) @@ -315,8 +334,8 @@ with_lock(void *baton, /* Enum identifying a filesystem lock. */ typedef enum lock_id_t { - write_lock, txn_lock, + write_lock, pack_lock } lock_id_t; @@ -332,12 +351,6 @@ init_lock_baton(with_lock_baton_t *baton, switch (lock_id) { - case write_lock: - baton->mutex = ffsd->fs_write_lock; - baton->lock_path = svn_fs_x__path_lock(baton->fs, baton->lock_pool); - baton->is_global_lock = TRUE; - break; - case txn_lock: baton->mutex = ffsd->txn_current_lock; baton->lock_path = svn_fs_x__path_txn_current_lock(baton->fs, @@ -345,6 +358,12 @@ init_lock_baton(with_lock_baton_t *baton, baton->is_global_lock = FALSE; break; + case write_lock: + baton->mutex = ffsd->fs_write_lock; + baton->lock_path = svn_fs_x__path_lock(baton->fs, baton->lock_pool); + baton->is_global_lock = TRUE; + break; + case pack_lock: baton->mutex = ffsd->fs_pack_lock; baton->lock_path = svn_fs_x__path_pack_lock(baton->fs, @@ -477,10 +496,10 @@ svn_fs_x__with_all_locks(svn_fs_t *fs, fs_fs_shared_data_t. The lock chain is being created in innermost (last to acquire) -> outermost (first to acquire) order. */ with_lock_baton_t *lock_baton - = create_lock_baton(fs, write_lock, body, baton, scratch_pool); + = create_lock_baton(fs, txn_lock, body, baton, scratch_pool); + lock_baton = chain_lock_baton(write_lock, lock_baton); lock_baton = chain_lock_baton(pack_lock, lock_baton); - lock_baton = chain_lock_baton(txn_lock, lock_baton); return svn_error_trace(with_lock(lock_baton, scratch_pool)); } @@ -702,7 +721,8 @@ get_writable_proto_rev(apr_file_t **file, /* Now open the prototype revision file and seek to the end. */ err = svn_io_file_open(file, svn_fs_x__path_txn_proto_rev(fs, txn_id, pool), - APR_WRITE | APR_BUFFERED, APR_OS_DEFAULT, pool); + APR_READ | APR_WRITE | APR_BUFFERED, APR_OS_DEFAULT, + pool); /* You might expect that we could dispense with the following seek and achieve the same thing by opening the file using APR_APPEND. @@ -858,16 +878,35 @@ unparse_dir_entry(svn_fs_x__dirent_t *dirent, svn_stream_t *stream, apr_pool_t *scratch_pool) { - const char *val - = apr_psprintf(scratch_pool, "%s %s", - (dirent->kind == svn_node_file) ? SVN_FS_X__KIND_FILE - : SVN_FS_X__KIND_DIR, - svn_fs_x__id_unparse(&dirent->id, scratch_pool)->data); + apr_size_t to_write; + apr_size_t name_len = strlen(dirent->name); + + /* A buffer with sufficient space for + * - entry name + 1 terminating NUL + * - 1 byte for the node kind + * - 2 numbers in 7b/8b encoding for the noderev-id + */ + apr_byte_t *buffer = apr_palloc(scratch_pool, + name_len + 2 + 2 * SVN__MAX_ENCODED_UINT_LEN); + + /* Now construct the value. */ + apr_byte_t *p = buffer; + + /* The entry name, terminated by NUL. */ + memcpy(p, dirent->name, name_len + 1); + p += name_len + 1; + + /* The entry type. */ + p = svn__encode_uint(p, dirent->kind); + + /* The ID. */ + p = svn__encode_int(p, dirent->id.change_set); + p = svn__encode_uint(p, dirent->id.number); + + /* Add the entry to the output stream. */ + to_write = p - buffer; + SVN_ERR(svn_stream_write(stream, (const char *)buffer, &to_write)); - SVN_ERR(svn_stream_printf(stream, scratch_pool, "K %" APR_SIZE_T_FMT - "\n%s\nV %" APR_SIZE_T_FMT "\n%s\n", - strlen(dirent->name), dirent->name, - strlen(val), val)); return SVN_NO_ERROR; } @@ -878,8 +917,15 @@ unparse_dir_entries(apr_array_header_t *entries, svn_stream_t *stream, apr_pool_t *scratch_pool) { + apr_byte_t buffer[SVN__MAX_ENCODED_UINT_LEN]; apr_pool_t *iterpool = svn_pool_create(scratch_pool); int i; + + /* Write the number of entries. */ + apr_size_t to_write = svn__encode_uint(buffer, entries->nelts) - buffer; + SVN_ERR(svn_stream_write(stream, (const char *)buffer, &to_write)); + + /* Write all entries */ for (i = 0; i < entries->nelts; ++i) { svn_fs_x__dirent_t *dirent; @@ -889,9 +935,6 @@ unparse_dir_entries(apr_array_header_t *entries, SVN_ERR(unparse_dir_entry(dirent, stream, iterpool)); } - SVN_ERR(svn_stream_printf(stream, scratch_pool, "%s\n", - SVN_HASH_TERMINATOR)); - svn_pool_destroy(iterpool); return SVN_NO_ERROR; } @@ -932,30 +975,10 @@ fold_change(apr_hash_t *changed_paths, /* This path already exists in the hash, so we have to merge this change into the already existing one. */ - /* Sanity check: only allow unused node revision IDs in the - `reset' case. */ - if ((! svn_fs_x__id_used(&change->noderev_id)) - && (change->change_kind != svn_fs_path_change_reset)) - return svn_error_create - (SVN_ERR_FS_CORRUPT, NULL, - _("Missing required node revision ID")); - - /* Sanity check: we should be talking about the same node - revision ID as our last change except where the last change - was a deletion. */ - if (svn_fs_x__id_used(&change->noderev_id) - && (!svn_fs_x__id_eq(&old_change->noderev_id, &change->noderev_id)) - && (old_change->change_kind != svn_fs_path_change_delete)) - return svn_error_create - (SVN_ERR_FS_CORRUPT, NULL, - _("Invalid change ordering: new node revision ID " - "without delete")); - /* Sanity check: an add, replacement, or reset must be the first thing to follow a deletion. */ if ((old_change->change_kind == svn_fs_path_change_delete) && (! ((change->change_kind == svn_fs_path_change_replace) - || (change->change_kind == svn_fs_path_change_reset) || (change->change_kind == svn_fs_path_change_add)))) return svn_error_create (SVN_ERR_FS_CORRUPT, NULL, @@ -964,8 +987,7 @@ fold_change(apr_hash_t *changed_paths, /* Sanity check: an add can't follow anything except a delete or reset. */ if ((change->change_kind == svn_fs_path_change_add) - && (old_change->change_kind != svn_fs_path_change_delete) - && (old_change->change_kind != svn_fs_path_change_reset)) + && (old_change->change_kind != svn_fs_path_change_delete)) return svn_error_create (SVN_ERR_FS_CORRUPT, NULL, _("Invalid change ordering: add change on preexisting path")); @@ -973,12 +995,6 @@ fold_change(apr_hash_t *changed_paths, /* Now, merge that change in. */ switch (change->change_kind) { - case svn_fs_path_change_reset: - /* A reset here will simply remove the path change from the - hash. */ - apr_hash_set(changed_paths, path->data, path->len, NULL); - break; - case svn_fs_path_change_delete: if (old_change->change_kind == svn_fs_path_change_add) { @@ -1207,34 +1223,65 @@ get_and_increment_txn_key_body(void *baton, apr_pool_t *scratch_pool) { get_and_increment_txn_key_baton_t *cb = baton; - const char *txn_current_filename = svn_fs_x__path_txn_current(cb->fs, - scratch_pool); - const char *tmp_filename; + svn_fs_t *fs = cb->fs; + apr_pool_t *iterpool = svn_pool_create(scratch_pool); + const char *txn_current_path = svn_fs_x__path_txn_current(fs, scratch_pool); char new_id_str[SVN_INT64_BUFFER_SIZE]; svn_stringbuf_t *buf; - SVN_ERR(svn_fs_x__read_content(&buf, txn_current_filename, scratch_pool)); + SVN_ERR(svn_fs_x__read_content(&buf, txn_current_path, scratch_pool)); - /* remove trailing newlines */ + /* Parse the txn number, stopping at the next non-digit. + * + * Note that an empty string is being interpreted as "0". + * This gives us implicit recovery if the file contents should be lost + * due to e.g. power failure. + */ cb->txn_number = svn__base36toui64(NULL, buf->data); + if (cb->txn_number == 0) + ++cb->txn_number; + + /* Check for conflicts. Those might happen if the server crashed and we + * had 'svnadmin recover' reset the txn counter. + * + * Once we found an unused txn id, claim it by creating the respective + * txn directory. + * + * Note that this is not racy because we hold the txn-current-lock. + */ + while (TRUE) + { + const char *txn_dir; + svn_node_kind_t kind; + svn_pool_clear(iterpool); + + txn_dir = svn_fs_x__path_txn_dir(fs, cb->txn_number, iterpool); + SVN_ERR(svn_io_check_path(txn_dir, &kind, iterpool)); + if (kind == svn_node_none) + { + svn_io_dir_make(txn_dir, APR_OS_DEFAULT, iterpool); + break; + } + + ++cb->txn_number; + } /* Increment the key and add a trailing \n to the string so the txn-current file has a newline in it. */ - SVN_ERR(svn_io_write_unique(&tmp_filename, - svn_dirent_dirname(txn_current_filename, - scratch_pool), - new_id_str, - svn__ui64tobase36(new_id_str, cb->txn_number+1), - svn_io_file_del_none, scratch_pool)); - SVN_ERR(svn_fs_x__move_into_place(tmp_filename, txn_current_filename, - txn_current_filename, scratch_pool)); + SVN_ERR(svn_io_write_atomic2(txn_current_path, new_id_str, + svn__ui64tobase36(new_id_str, + cb->txn_number + 1), + txn_current_path, FALSE, scratch_pool)); + + svn_pool_destroy(iterpool); return SVN_NO_ERROR; } /* Create a unique directory for a transaction in FS based on revision REV. - Return the ID for this transaction in *ID_P and *TXN_ID. Use a sequence - value in the transaction ID to prevent reuse of transaction IDs. */ + Return the ID for this transaction in *ID_P, allocated from RESULT_POOL + and *TXN_ID. Use a sequence value in the transaction ID to prevent reuse + of transaction IDs. Allocate temporaries from SCRATCH_POOL. */ static svn_error_t * create_txn_dir(const char **id_p, svn_fs_x__txn_id_t *txn_id, @@ -1243,23 +1290,20 @@ create_txn_dir(const char **id_p, apr_pool_t *scratch_pool) { get_and_increment_txn_key_baton_t cb; - const char *txn_dir; /* Get the current transaction sequence value, which is a base-36 - number, from the txn-current file, and write an - incremented value back out to the file. Place the revision - number the transaction is based off into the transaction id. */ + number, from the txn-current file, and write an + incremented value back out to the file. Place the revision + number the transaction is based off into the transaction id. */ cb.fs = fs; SVN_ERR(svn_fs_x__with_txn_current_lock(fs, get_and_increment_txn_key_body, &cb, scratch_pool)); *txn_id = cb.txn_number; - *id_p = svn_fs_x__txn_name(*txn_id, result_pool); - txn_dir = svn_fs_x__path_txn_dir(fs, *txn_id, scratch_pool); - return svn_io_dir_make(txn_dir, APR_OS_DEFAULT, scratch_pool); + return SVN_NO_ERROR; } /* Create a new transaction in filesystem FS, based on revision REV, @@ -1321,15 +1365,16 @@ create_txn(svn_fs_txn_t **txn_p, return SVN_NO_ERROR; } -/* Store the property list for transaction TXN_ID in PROPLIST. - Perform temporary allocations in POOL. */ +/* Store the property list for transaction TXN_ID in *PROPLIST, allocated + from RESULT_POOL. Perform temporary allocations in SCRATCH_POOL. */ static svn_error_t * -get_txn_proplist(apr_hash_t *proplist, +get_txn_proplist(apr_hash_t **proplist, svn_fs_t *fs, svn_fs_x__txn_id_t txn_id, - apr_pool_t *pool) + apr_pool_t *result_pool, + apr_pool_t *scratch_pool) { - svn_stream_t *stream; + svn_stringbuf_t *content; /* Check for issue #3696. (When we find and fix the cause, we can change * this to an assertion.) */ @@ -1339,14 +1384,20 @@ get_txn_proplist(apr_hash_t *proplist, "passed to get_txn_proplist()")); /* Open the transaction properties file. */ - SVN_ERR(svn_stream_open_readonly(&stream, - svn_fs_x__path_txn_props(fs, txn_id, pool), - pool, pool)); + SVN_ERR(svn_stringbuf_from_file2(&content, + svn_fs_x__path_txn_props(fs, txn_id, + scratch_pool), + result_pool)); /* Read in the property list. */ - SVN_ERR(svn_hash_read2(proplist, stream, SVN_HASH_TERMINATOR, pool)); + SVN_ERR_W(svn_fs_x__parse_properties(proplist, + svn_stringbuf__morph_into_string(content), + result_pool), + apr_psprintf(scratch_pool, + _("malformed property list in transaction '%s'"), + svn_fs_x__path_txn_props(fs, txn_id, scratch_pool))); - return svn_stream_close(stream); + return SVN_NO_ERROR; } /* Save the property list PROPS as the revprops for transaction TXN_ID @@ -1355,26 +1406,27 @@ static svn_error_t * set_txn_proplist(svn_fs_t *fs, svn_fs_x__txn_id_t txn_id, apr_hash_t *props, - svn_boolean_t final, apr_pool_t *scratch_pool) { - svn_stringbuf_t *buf; svn_stream_t *stream; - - /* Write out the new file contents to BUF. */ - buf = svn_stringbuf_create_ensure(1024, scratch_pool); - stream = svn_stream_from_stringbuf(buf, scratch_pool); - SVN_ERR(svn_hash_write2(props, stream, SVN_HASH_TERMINATOR, scratch_pool)); + const char *temp_path; + + /* Write the new contents into a temporary file. */ + SVN_ERR(svn_stream_open_unique(&stream, &temp_path, + svn_fs_x__path_txn_dir(fs, txn_id, + scratch_pool), + svn_io_file_del_none, + scratch_pool, scratch_pool)); + SVN_ERR(svn_fs_x__write_properties(stream, props, scratch_pool)); SVN_ERR(svn_stream_close(stream)); - /* Open the transaction properties file and write new contents to it. */ - SVN_ERR(svn_io_write_atomic((final - ? svn_fs_x__path_txn_props_final(fs, txn_id, - scratch_pool) - : svn_fs_x__path_txn_props(fs, txn_id, - scratch_pool)), - buf->data, buf->len, - NULL /* copy_perms_path */, scratch_pool)); + /* Replace the old file with the new one. */ + SVN_ERR(svn_io_file_rename2(temp_path, + svn_fs_x__path_txn_props(fs, txn_id, + scratch_pool), + FALSE, + scratch_pool)); + return SVN_NO_ERROR; } @@ -1402,11 +1454,12 @@ svn_fs_x__change_txn_props(svn_fs_txn_t *txn, apr_pool_t *scratch_pool) { fs_txn_data_t *ftd = txn->fsap_data; - apr_hash_t *txn_prop = apr_hash_make(scratch_pool); + apr_pool_t *subpool = svn_pool_create(scratch_pool); + apr_hash_t *txn_prop; int i; svn_error_t *err; - err = get_txn_proplist(txn_prop, txn->fs, ftd->txn_id, scratch_pool); + err = get_txn_proplist(&txn_prop, txn->fs, ftd->txn_id, subpool, subpool); /* Here - and here only - we need to deal with the possibility that the transaction property file doesn't yet exist. The rest of the implementation assumes that the file exists, but we're called to set the @@ -1423,16 +1476,16 @@ svn_fs_x__change_txn_props(svn_fs_txn_t *txn, if (svn_hash_gets(txn_prop, SVN_FS__PROP_TXN_CLIENT_DATE) && !strcmp(prop->name, SVN_PROP_REVISION_DATE)) svn_hash_sets(txn_prop, SVN_FS__PROP_TXN_CLIENT_DATE, - svn_string_create("1", scratch_pool)); + svn_string_create("1", subpool)); svn_hash_sets(txn_prop, prop->name, prop->value); } /* Create a new version of the file and write out the new props. */ /* Open the transaction properties file. */ - SVN_ERR(set_txn_proplist(txn->fs, ftd->txn_id, txn_prop, FALSE, - scratch_pool)); + SVN_ERR(set_txn_proplist(txn->fs, ftd->txn_id, txn_prop, subpool)); + svn_pool_destroy(subpool); return SVN_NO_ERROR; } @@ -1447,9 +1500,6 @@ svn_fs_x__get_txn(svn_fs_x__transaction_t **txn_p, svn_fs_x__id_t root_id; txn = apr_pcalloc(pool, sizeof(*txn)); - txn->proplist = apr_hash_make(pool); - - SVN_ERR(get_txn_proplist(txn->proplist, fs, txn_id, pool)); svn_fs_x__init_txn_root(&root_id, txn_id); SVN_ERR(svn_fs_x__get_node_revision(&noderev, fs, &root_id, pool, pool)); @@ -1462,8 +1512,8 @@ svn_fs_x__get_txn(svn_fs_x__transaction_t **txn_p, return SVN_NO_ERROR; } -/* If it is supported by the format of file system FS, store the (ITEM_INDEX, - * OFFSET) pair in the log-to-phys proto index file of transaction TXN_ID. +/* Store the (ITEM_INDEX, OFFSET) pair in the log-to-phys proto index file + * of transaction TXN_ID in filesystem FS. * Use SCRATCH_POOL for temporary allocations. */ static svn_error_t * @@ -1483,14 +1533,13 @@ store_l2p_index_entry(svn_fs_t *fs, return SVN_NO_ERROR; } -/* If it is supported by the format of file system FS, store ENTRY in the - * phys-to-log proto index file of transaction TXN_ID. - * Use SCRATCH_POOL for temporary allocations. +/* Store ENTRY in the phys-to-log proto index file of transaction TXN_ID + * in filesystem FS. Use SCRATCH_POOL for temporary allocations. */ static svn_error_t * store_p2l_index_entry(svn_fs_t *fs, svn_fs_x__txn_id_t txn_id, - svn_fs_x__p2l_entry_t *entry, + const svn_fs_x__p2l_entry_t *entry, apr_pool_t *scratch_pool) { const char *path = svn_fs_x__path_p2l_proto_index(fs, txn_id, scratch_pool); @@ -1515,19 +1564,24 @@ allocate_item_index(apr_uint64_t *item_index, char buffer[SVN_INT64_BUFFER_SIZE] = { 0 }; svn_boolean_t eof = FALSE; apr_size_t to_write; - apr_size_t read; + apr_size_t bytes_read; apr_off_t offset = 0; /* read number */ SVN_ERR(svn_io_file_open(&file, svn_fs_x__path_txn_item_index(fs, txn_id, scratch_pool), - APR_READ | APR_WRITE - | APR_CREATE | APR_BUFFERED, + APR_READ | APR_WRITE | APR_CREATE, APR_OS_DEFAULT, scratch_pool)); SVN_ERR(svn_io_file_read_full2(file, buffer, sizeof(buffer)-1, - &read, &eof, scratch_pool)); - if (read) + &bytes_read, &eof, scratch_pool)); + + /* Item index file should be shorter than SVN_INT64_BUFFER_SIZE, + otherwise we truncate data. */ + if (!eof) + return svn_error_create(SVN_ERR_FS_CORRUPT, NULL, + _("Unexpected itemidx file length")); + else if (bytes_read) SVN_ERR(svn_cstring_atoui64(item_index, buffer)); else *item_index = SVN_FS_X__ITEM_INDEX_FIRST_USER; @@ -1683,25 +1737,30 @@ svn_fs_x__purge_txn(svn_fs_t *fs, apr_pool_t *scratch_pool) { svn_fs_x__txn_id_t txn_id; + + /* The functions we are calling open files and operate on the OS FS. + Since these may allocate a non-trivial amount of memory, do that + in a SUBPOOL and clear that one up before returning. */ + apr_pool_t *subpool = svn_pool_create(scratch_pool); SVN_ERR(svn_fs_x__txn_by_name(&txn_id, txn_id_str)); /* Remove the shared transaction object associated with this transaction. */ - SVN_ERR(purge_shared_txn(fs, txn_id, scratch_pool)); + SVN_ERR(purge_shared_txn(fs, txn_id, subpool)); /* Remove the directory associated with this transaction. */ - SVN_ERR(svn_io_remove_dir2(svn_fs_x__path_txn_dir(fs, txn_id, scratch_pool), - FALSE, NULL, NULL, scratch_pool)); + SVN_ERR(svn_io_remove_dir2(svn_fs_x__path_txn_dir(fs, txn_id, subpool), + FALSE, NULL, NULL, subpool)); - /* Delete protorev and its lock, which aren't in the txn - directory. It's OK if they don't exist (for example, if this - is post-commit and the proto-rev has been moved into - place). */ + /* Delete protorev and its lock, which aren't in the txn directory. + It's OK if they don't exist (for example, if this is post-commit + and the proto-rev has been moved into place). */ SVN_ERR(svn_io_remove_file2( - svn_fs_x__path_txn_proto_rev(fs, txn_id, scratch_pool), - TRUE, scratch_pool)); + svn_fs_x__path_txn_proto_rev(fs, txn_id, subpool), + TRUE, subpool)); SVN_ERR(svn_io_remove_file2( - svn_fs_x__path_txn_proto_rev_lock(fs, txn_id, scratch_pool), - TRUE, scratch_pool)); + svn_fs_x__path_txn_proto_rev_lock(fs, txn_id, subpool), + TRUE, subpool)); + svn_pool_destroy(subpool); return SVN_NO_ERROR; } @@ -1736,12 +1795,16 @@ svn_fs_x__set_entry(svn_fs_t *fs, scratch_pool, scratch_pool); apr_file_t *file; svn_stream_t *out; + svn_filesize_t filesize; svn_fs_x__data_t *ffd = fs->fsap_data; apr_pool_t *subpool = svn_pool_create(scratch_pool); + const svn_fs_x__id_t *key = &(parent_noderev->noderev_id); + svn_fs_x__dirent_t entry; if (!rep || !svn_fs_x__is_txn(rep->id.change_set)) { apr_array_header_t *entries; + svn_fs_x__dir_data_t dir_data; /* Before we can modify the directory, we need to dump its old contents into a mutable representation file. */ @@ -1753,8 +1816,6 @@ svn_fs_x__set_entry(svn_fs_t *fs, out = svn_stream_from_aprfile2(file, TRUE, scratch_pool); SVN_ERR(unparse_dir_entries(entries, out, subpool)); - svn_pool_clear(subpool); - /* Provide the parent with a data rep if it had none before (directories so far empty). */ if (!rep) @@ -1769,23 +1830,86 @@ svn_fs_x__set_entry(svn_fs_t *fs, /* Save noderev to disk. */ SVN_ERR(svn_fs_x__put_node_revision(fs, parent_noderev, subpool)); + + /* Immediately populate the txn dir cache to avoid re-reading + * the file we just wrote. */ + + /* Flush APR buffers. */ + SVN_ERR(svn_io_file_flush(file, subpool)); + + /* Obtain final file size to update txn_dir_cache. */ + SVN_ERR(svn_io_file_size_get(&filesize, file, subpool)); + + /* Store in the cache. */ + dir_data.entries = entries; + dir_data.txn_filesize = filesize; + SVN_ERR(svn_cache__set(ffd->dir_cache, key, &dir_data, subpool)); + + svn_pool_clear(subpool); } else { + svn_boolean_t found; + svn_filesize_t cached_filesize; + /* The directory rep is already mutable, so just open it for append. */ SVN_ERR(svn_io_file_open(&file, filename, APR_WRITE | APR_APPEND, - APR_OS_DEFAULT, scratch_pool)); - out = svn_stream_from_aprfile2(file, TRUE, scratch_pool); + APR_OS_DEFAULT, subpool)); + out = svn_stream_from_aprfile2(file, TRUE, subpool); + + /* If the cache contents is stale, drop it. + * + * Note that the directory file is append-only, i.e. if the size + * did not change, the contents didn't either. */ + + /* Get the file size that corresponds to the cached contents + * (if any). */ + SVN_ERR(svn_cache__get_partial((void **)&cached_filesize, &found, + ffd->dir_cache, key, + svn_fs_x__extract_dir_filesize, + NULL, subpool)); + + /* File size info still matches? + * If not, we need to drop the cache entry. */ + if (found) + { + SVN_ERR(svn_io_file_size_get(&filesize, file, subpool)); + + if (cached_filesize != filesize) + SVN_ERR(svn_cache__set(ffd->dir_cache, key, NULL, subpool)); + } } + /* Append an incremental hash entry for the entry change. + A deletion is represented by an "unused" noderev-id. */ + if (id) + entry.id = *id; + else + svn_fs_x__id_reset(&entry.id); + + entry.name = name; + entry.kind = kind; + + SVN_ERR(unparse_dir_entry(&entry, out, subpool)); + + /* Flush APR buffers. */ + SVN_ERR(svn_io_file_flush(file, subpool)); + + /* Obtain final file size to update txn_dir_cache. */ + SVN_ERR(svn_io_file_size_get(&filesize, file, subpool)); + + /* Close file. */ + SVN_ERR(svn_io_file_close(file, subpool)); + svn_pool_clear(subpool); + /* update directory cache */ { - /* build parameters: (name, new entry) pair */ - const svn_fs_x__id_t *key = &(parent_noderev->noderev_id); + /* build parameters: name, new entry, new file size */ replace_baton_t baton; baton.name = name; baton.new_entry = NULL; + baton.txn_filesize = filesize; if (id) { @@ -1800,25 +1924,7 @@ svn_fs_x__set_entry(svn_fs_t *fs, svn_fs_x__replace_dir_entry, &baton, subpool)); } - svn_pool_clear(subpool); - - /* Append an incremental hash entry for the entry change. */ - if (id) - { - svn_fs_x__dirent_t entry; - entry.name = name; - entry.id = *id; - entry.kind = kind; - - SVN_ERR(unparse_dir_entry(&entry, out, subpool)); - } - else - { - SVN_ERR(svn_stream_printf(out, subpool, "D %" APR_SIZE_T_FMT "\n%s\n", - strlen(name), name)); - } - SVN_ERR(svn_io_file_close(file, subpool)); svn_pool_destroy(subpool); return SVN_NO_ERROR; } @@ -1827,7 +1933,6 @@ svn_error_t * svn_fs_x__add_change(svn_fs_t *fs, svn_fs_x__txn_id_t txn_id, const char *path, - const svn_fs_x__id_t *id, svn_fs_path_change_kind_t change_kind, svn_boolean_t text_mod, svn_boolean_t prop_mod, @@ -1850,7 +1955,6 @@ svn_fs_x__add_change(svn_fs_t *fs, change.path.data = path; change.path.len = strlen(path); - change.noderev_id = *id; change.change_kind = change_kind; change.text_mod = text_mod; change.prop_mod = prop_mod; @@ -1975,8 +2079,7 @@ shards_spanned(int *spanned, /* Given a node-revision NODEREV in filesystem FS, return the representation in *REP to use as the base for a text representation delta if PROPS is FALSE. If PROPS has been set, a suitable props - base representation will be returned. Perform temporary allocations - in *POOL. */ + base representation will be returned. Perform allocations in POOL. */ static svn_error_t * choose_delta_base(svn_fs_x__representation_t **rep, svn_fs_t *fs, @@ -1984,9 +2087,10 @@ choose_delta_base(svn_fs_x__representation_t **rep, svn_boolean_t props, apr_pool_t *pool) { - /* The zero-based index (counting from the "oldest" end), along NODEREVs line - * predecessors, of the node-rev we will use as delta base. */ + /* The zero-based index (counting from the "oldest" end), along NODEREVs + * line predecessors, of the node-rev we will use as delta base. */ int count; + /* The length of the linear part of a delta chain. (Delta chains use * skip-delta bits for the high-order bits and are linear in the low-order * bits.) */ @@ -2175,7 +2279,7 @@ rep_write_get_baton(rep_write_baton_t **wb_p, b->local_pool), b->local_pool); - SVN_ERR(svn_fs_x__get_file_offset(&b->rep_offset, file, b->local_pool)); + SVN_ERR(svn_io_file_get_offset(&b->rep_offset, file, b->local_pool)); /* Get the base for this delta. */ SVN_ERR(choose_delta_base(&base_rep, fs, noderev, FALSE, b->local_pool)); @@ -2195,11 +2299,10 @@ rep_write_get_baton(rep_write_baton_t **wb_p, header.type = svn_fs_x__rep_self_delta; } SVN_ERR(svn_fs_x__write_rep_header(&header, b->rep_stream, - b->local_pool)); + b->local_pool)); /* Now determine the offset of the actual svndiff data. */ - SVN_ERR(svn_fs_x__get_file_offset(&b->delta_start, file, - b->local_pool)); + SVN_ERR(svn_io_file_get_offset(&b->delta_start, file, b->local_pool)); /* Cleanup in case something goes wrong. */ apr_pool_cleanup_register(b->local_pool, b, rep_write_cleanup, @@ -2222,18 +2325,26 @@ rep_write_get_baton(rep_write_baton_t **wb_p, } /* For REP->SHA1_CHECKSUM, try to find an already existing representation - in FS and return it in *OUT_REP. If no such representation exists or + in FS and return it in *OLD_REP. If no such representation exists or if rep sharing has been disabled for FS, NULL will be returned. Since there may be new duplicate representations within the same uncommitted revision, those can be passed in REPS_HASH (maps a sha1 digest onto svn_fs_x__representation_t*), otherwise pass in NULL for REPS_HASH. + + The content of both representations will be compared, taking REP's content + from FILE at OFFSET. Only if they actually match, will *OLD_REP not be + NULL. + Use RESULT_POOL for *OLD_REP allocations and SCRATCH_POOL for temporaries. The lifetime of *OLD_REP is limited by both, RESULT_POOL and REP lifetime. */ static svn_error_t * get_shared_rep(svn_fs_x__representation_t **old_rep, svn_fs_t *fs, + svn_fs_x__txn_id_t txn_id, svn_fs_x__representation_t *rep, + apr_file_t *file, + apr_off_t offset, apr_hash_t *reps_hash, apr_pool_t *result_pool, apr_pool_t *scratch_pool) @@ -2241,14 +2352,22 @@ get_shared_rep(svn_fs_x__representation_t **old_rep, svn_error_t *err; svn_fs_x__data_t *ffd = fs->fsap_data; + svn_checksum_t checksum; + checksum.digest = rep->sha1_digest; + checksum.kind = svn_checksum_sha1; + /* Return NULL, if rep sharing has been disabled. */ *old_rep = NULL; if (!ffd->rep_sharing_allowed) return SVN_NO_ERROR; + /* Can't look up if we don't know the key (happens for directories). */ + if (!rep->has_sha1) + return SVN_NO_ERROR; + /* Check and see if we already have a representation somewhere that's identical to the one we just wrote out. Start with the hash lookup - because it is cheepest. */ + because it is cheapest. */ if (reps_hash) *old_rep = apr_hash_get(reps_hash, rep->sha1_digest, @@ -2257,9 +2376,6 @@ get_shared_rep(svn_fs_x__representation_t **old_rep, /* If we haven't found anything yet, try harder and consult our DB. */ if (*old_rep == NULL) { - svn_checksum_t checksum; - checksum.digest = rep->sha1_digest; - checksum.kind = svn_checksum_sha1; err = svn_fs_x__get_rep_reference(old_rep, fs, &checksum, result_pool, scratch_pool); @@ -2318,17 +2434,125 @@ get_shared_rep(svn_fs_x__representation_t **old_rep, } } - /* Add information that is missing in the cached data. */ - if (*old_rep) + if (!*old_rep) + return SVN_NO_ERROR; + + /* A simple guard against general rep-cache induced corruption. */ + if ((*old_rep)->expanded_size != rep->expanded_size) { - /* Use the old rep for this content. */ + /* Make the problem show up in the server log. + + Because not sharing reps is always a safe option, + terminating the request would be inappropriate. + */ + err = svn_error_createf(SVN_ERR_FS_CORRUPT, NULL, + "Rep size %s mismatches rep-cache.db value %s " + "for SHA1 %s.\n" + "You should delete the rep-cache.db and " + "verify the repository. The cached rep will " + "not be shared.", + apr_psprintf(scratch_pool, + "%" SVN_FILESIZE_T_FMT, + rep->expanded_size), + apr_psprintf(scratch_pool, + "%" SVN_FILESIZE_T_FMT, + (*old_rep)->expanded_size), + svn_checksum_to_cstring_display(&checksum, + scratch_pool)); + + (fs->warning)(fs->warning_baton, err); + svn_error_clear(err); + + /* Ignore the shared rep. */ + *old_rep = NULL; + } + else + { + /* Add information that is missing in the cached data. + Use the old rep for this content. */ memcpy((*old_rep)->md5_digest, rep->md5_digest, sizeof(rep->md5_digest)); } + /* If we (very likely) found a matching representation, compare the actual + * contents such that we can be sure that no rep-cache.db corruption or + * hash collision produced a false positive. */ + if (*old_rep) + { + apr_off_t old_position; + svn_stream_t *contents; + svn_stream_t *old_contents; + svn_boolean_t same; + + /* Make sure we can later restore FILE's current position. */ + SVN_ERR(svn_io_file_get_offset(&old_position, file, scratch_pool)); + + /* Compare the two representations. + * Note that the stream comparison might also produce MD5 checksum + * errors or other failures in case of SHA1 collisions. */ + SVN_ERR(svn_fs_x__get_contents_from_file(&contents, fs, rep, file, + offset, scratch_pool)); + if ((*old_rep)->id.change_set == rep->id.change_set) + { + /* Comparing with contents from the same transaction means + * reading the same prote-rev FILE. In the commit stage, + * the file will already have been moved and the IDs already + * bumped to the final revision. Hence, we must determine + * the OFFSET "manually". */ + svn_fs_x__revision_file_t *rev_file; + apr_uint32_t sub_item = 0; + svn_fs_x__id_t id; + id.change_set = svn_fs_x__change_set_by_txn(txn_id); + id.number = (*old_rep)->id.number; + + SVN_ERR(svn_fs_x__rev_file_wrap_temp(&rev_file, fs, file, + scratch_pool)); + SVN_ERR(svn_fs_x__item_offset(&offset, &sub_item, fs, rev_file, + &id, scratch_pool)); + + SVN_ERR(svn_fs_x__get_contents_from_file(&old_contents, fs, + *old_rep, file, + offset, scratch_pool)); + } + else + { + SVN_ERR(svn_fs_x__get_contents(&old_contents, fs, *old_rep, + FALSE, scratch_pool)); + } + err = svn_stream_contents_same2(&same, contents, old_contents, + scratch_pool); + + /* A mismatch should be extremely rare. + * If it does happen, reject the commit. */ + if (!same || err) + { + /* SHA1 collision or worse. */ + svn_stringbuf_t *old_rep_str + = svn_fs_x__unparse_representation(*old_rep, FALSE, + scratch_pool, + scratch_pool); + svn_stringbuf_t *rep_str + = svn_fs_x__unparse_representation(rep, FALSE, + scratch_pool, + scratch_pool); + const char *checksum__str + = svn_checksum_to_cstring_display(&checksum, scratch_pool); + + return svn_error_createf(SVN_ERR_FS_AMBIGUOUS_CHECKSUM_REP, + err, "SHA1 of reps '%s' and '%s' " + "matches (%s) but contents differ", + old_rep_str->data, rep_str->data, + checksum__str); + } + + /* Restore FILE's read / write position. */ + SVN_ERR(svn_io_file_seek(file, APR_SET, &old_position, scratch_pool)); + } + return SVN_NO_ERROR; } /* Copy the hash sum calculation results from MD5_CTX, SHA1_CTX into REP. + * SHA1 results are only be set if SHA1_CTX is not NULL. * Use SCRATCH_POOL for temporary allocations. */ static svn_error_t * @@ -2341,10 +2565,12 @@ digests_final(svn_fs_x__representation_t *rep, SVN_ERR(svn_checksum_final(&checksum, md5_ctx, scratch_pool)); memcpy(rep->md5_digest, checksum->digest, svn_checksum_size(checksum)); - SVN_ERR(svn_checksum_final(&checksum, sha1_ctx, scratch_pool)); - rep->has_sha1 = checksum != NULL; + rep->has_sha1 = sha1_ctx != NULL; if (rep->has_sha1) - memcpy(rep->sha1_digest, checksum->digest, svn_checksum_size(checksum)); + { + SVN_ERR(svn_checksum_final(&checksum, sha1_ctx, scratch_pool)); + memcpy(rep->sha1_digest, checksum->digest, svn_checksum_size(checksum)); + } return SVN_NO_ERROR; } @@ -2368,7 +2594,7 @@ rep_write_contents_close(void *baton) SVN_ERR(svn_stream_close(b->delta_stream)); /* Determine the length of the svndiff data. */ - SVN_ERR(svn_fs_x__get_file_offset(&offset, b->file, b->local_pool)); + SVN_ERR(svn_io_file_get_offset(&offset, b->file, b->local_pool)); rep->size = offset - b->delta_start; /* Fill in the rest of the representation field. */ @@ -2382,8 +2608,8 @@ rep_write_contents_close(void *baton) /* Check and see if we already have a representation somewhere that's identical to the one we just wrote out. */ - SVN_ERR(get_shared_rep(&old_rep, b->fs, rep, NULL, b->result_pool, - b->local_pool)); + SVN_ERR(get_shared_rep(&old_rep, b->fs, txn_id, rep, b->file, b->rep_offset, + NULL, b->result_pool, b->local_pool)); if (old_rep) { @@ -2420,18 +2646,23 @@ rep_write_contents_close(void *baton) noderev_id.number = rep->id.number; entry.offset = b->rep_offset; - SVN_ERR(svn_fs_x__get_file_offset(&offset, b->file, b->local_pool)); + SVN_ERR(svn_io_file_get_offset(&offset, b->file, b->local_pool)); entry.size = offset - b->rep_offset; entry.type = SVN_FS_X__ITEM_TYPE_FILE_REP; entry.item_count = 1; entry.items = &noderev_id; entry.fnv1_checksum = b->fnv1a_checksum; - SVN_ERR(store_sha1_rep_mapping(b->fs, b->noderev, b->local_pool)); SVN_ERR(store_p2l_index_entry(b->fs, txn_id, &entry, b->local_pool)); } SVN_ERR(svn_io_file_close(b->file, b->local_pool)); + + /* Write the sha1->rep mapping *after* we successfully written node + * revision to disk. */ + if (!old_rep) + SVN_ERR(store_sha1_rep_mapping(b->fs, b->noderev, b->local_pool)); + SVN_ERR(unlock_proto_rev(b->fs, txn_id, b->lockcookie, b->local_pool)); svn_pool_destroy(b->local_pool); @@ -2520,7 +2751,7 @@ svn_fs_x__set_proplist(svn_fs_t *fs, APR_WRITE | APR_CREATE | APR_TRUNCATE | APR_BUFFERED, APR_OS_DEFAULT, scratch_pool)); out = svn_stream_from_aprfile2(file, TRUE, scratch_pool); - SVN_ERR(svn_hash_write2(proplist, out, SVN_HASH_TERMINATOR, scratch_pool)); + SVN_ERR(svn_fs_x__write_properties(out, proplist, scratch_pool)); SVN_ERR(svn_io_file_close(file, scratch_pool)); /* Mark the node-rev's prop rep as mutable, if not already done. */ @@ -2529,7 +2760,8 @@ svn_fs_x__set_proplist(svn_fs_t *fs, { svn_fs_x__txn_id_t txn_id = svn_fs_x__get_txn_id(noderev->noderev_id.change_set); - noderev->prop_rep = apr_pcalloc(scratch_pool, sizeof(*noderev->prop_rep)); + noderev->prop_rep = apr_pcalloc(scratch_pool, + sizeof(*noderev->prop_rep)); noderev->prop_rep->id.change_set = id->change_set; SVN_ERR(allocate_item_index(&noderev->prop_rep->id.number, fs, txn_id, scratch_pool)); @@ -2547,6 +2779,8 @@ typedef struct write_container_baton_t apr_size_t size; svn_checksum_ctx_t *md5_ctx; + + /* SHA1 calculation is optional. If not needed, this will be NULL. */ svn_checksum_ctx_t *sha1_ctx; } write_container_baton_t; @@ -2561,7 +2795,8 @@ write_container_handler(void *baton, write_container_baton_t *whb = baton; SVN_ERR(svn_checksum_update(whb->md5_ctx, data, *len)); - SVN_ERR(svn_checksum_update(whb->sha1_ctx, data, *len)); + if (whb->sha1_ctx) + SVN_ERR(svn_checksum_update(whb->sha1_ctx, data, *len)); SVN_ERR(svn_stream_write(whb->stream, data, len)); whb->size += *len; @@ -2583,7 +2818,7 @@ write_hash_to_stream(svn_stream_t *stream, apr_pool_t *scratch_pool) { apr_hash_t *hash = baton; - SVN_ERR(svn_hash_write2(hash, stream, SVN_HASH_TERMINATOR, scratch_pool)); + SVN_ERR(svn_fs_x__write_properties(stream, hash, scratch_pool)); return SVN_NO_ERROR; } @@ -2605,11 +2840,14 @@ write_directory_to_stream(svn_stream_t *stream, /* Write out the COLLECTION pertaining to the NODEREV in FS as a deltified text representation to file FILE using WRITER. In the process, record the total size and the md5 digest in REP and add the representation of type - ITEM_TYPE to the indexes if necessary. If rep sharing has been enabled and - REPS_HASH is not NULL, it will be used in addition to the on-disk cache to - find earlier reps with the same content. When such existing reps can be - found, we will truncate the one just written from the file and return the - existing rep. + ITEM_TYPE to the indexes if necessary. + + If ALLOW_REP_SHARING is FALSE, rep-sharing will not be used, regardless + of any other option and rep-sharing settings. If rep sharing has been + enabled and REPS_HASH is not NULL, it will be used in addition to the + on-disk cache to find earlier reps with the same content. If such + existing reps can be found, we will truncate the one just written from + the file and return the existing rep. If ITEM_TYPE is IS_PROPS equals SVN_FS_FS__ITEM_TYPE_*_PROPS, assume that we want to a props representation as the base for our delta. @@ -2626,6 +2864,7 @@ write_container_delta_rep(svn_fs_x__representation_t *rep, svn_fs_x__txn_id_t txn_id, svn_fs_x__noderev_t *noderev, apr_hash_t *reps_hash, + svn_boolean_t allow_rep_sharing, apr_uint32_t item_type, svn_revnum_t final_revision, apr_pool_t *scratch_pool) @@ -2637,7 +2876,7 @@ write_container_delta_rep(svn_fs_x__representation_t *rep, svn_stream_t *file_stream; svn_stream_t *stream; svn_fs_x__representation_t *base_rep; - svn_fs_x__representation_t *old_rep; + svn_fs_x__representation_t *old_rep = NULL; svn_fs_x__p2l_entry_t entry; svn_stream_t *source; svn_fs_x__rep_header_t header = { 0 }; @@ -2655,7 +2894,7 @@ write_container_delta_rep(svn_fs_x__representation_t *rep, SVN_ERR(choose_delta_base(&base_rep, fs, noderev, is_props, scratch_pool)); SVN_ERR(svn_fs_x__get_contents(&source, fs, base_rep, FALSE, scratch_pool)); - SVN_ERR(svn_fs_x__get_file_offset(&offset, file, scratch_pool)); + SVN_ERR(svn_io_file_get_offset(&offset, file, scratch_pool)); /* Write out the rep header. */ if (base_rep) @@ -2676,7 +2915,7 @@ write_container_delta_rep(svn_fs_x__representation_t *rep, scratch_pool), scratch_pool); SVN_ERR(svn_fs_x__write_rep_header(&header, file_stream, scratch_pool)); - SVN_ERR(svn_fs_x__get_file_offset(&delta_start, file, scratch_pool)); + SVN_ERR(svn_io_file_get_offset(&delta_start, file, scratch_pool)); /* Prepare to write the svndiff data. */ svn_txdelta_to_svndiff3(&diff_wh, @@ -2691,7 +2930,8 @@ write_container_delta_rep(svn_fs_x__representation_t *rep, scratch_pool); whb->size = 0; whb->md5_ctx = svn_checksum_ctx_create(svn_checksum_md5, scratch_pool); - whb->sha1_ctx = svn_checksum_ctx_create(svn_checksum_sha1, scratch_pool); + if (item_type != SVN_FS_X__ITEM_TYPE_DIR_REP) + whb->sha1_ctx = svn_checksum_ctx_create(svn_checksum_sha1, scratch_pool); /* serialize the hash */ stream = svn_stream_create(whb, scratch_pool); @@ -2703,10 +2943,16 @@ write_container_delta_rep(svn_fs_x__representation_t *rep, /* Store the results. */ SVN_ERR(digests_final(rep, whb->md5_ctx, whb->sha1_ctx, scratch_pool)); + /* Update size info. */ + SVN_ERR(svn_io_file_get_offset(&rep_end, file, scratch_pool)); + rep->size = rep_end - delta_start; + rep->expanded_size = whb->size; + /* Check and see if we already have a representation somewhere that's identical to the one we just wrote out. */ - SVN_ERR(get_shared_rep(&old_rep, fs, rep, reps_hash, scratch_pool, - scratch_pool)); + if (allow_rep_sharing) + SVN_ERR(get_shared_rep(&old_rep, fs, txn_id, rep, file, offset, reps_hash, + scratch_pool, scratch_pool)); if (old_rep) { @@ -2723,7 +2969,6 @@ write_container_delta_rep(svn_fs_x__representation_t *rep, svn_fs_x__id_t noderev_id; /* Write out our cosmetic end marker. */ - SVN_ERR(svn_fs_x__get_file_offset(&rep_end, file, scratch_pool)); SVN_ERR(svn_stream_puts(file_stream, "ENDREP\n")); SVN_ERR(svn_stream_close(file_stream)); @@ -2736,7 +2981,7 @@ write_container_delta_rep(svn_fs_x__representation_t *rep, noderev_id.number = rep->id.number; entry.offset = offset; - SVN_ERR(svn_fs_x__get_file_offset(&offset, file, scratch_pool)); + SVN_ERR(svn_io_file_get_offset(&offset, file, scratch_pool)); entry.size = offset - entry.offset; entry.type = item_type; entry.item_count = 1; @@ -2745,7 +2990,6 @@ write_container_delta_rep(svn_fs_x__representation_t *rep, SVN_ERR(store_p2l_index_entry(fs, txn_id, &entry, scratch_pool)); /* update the representation */ - rep->expanded_size = whb->size; rep->size = rep_end - delta_start; } @@ -2791,7 +3035,7 @@ validate_root_noderev(svn_fs_t *fs, Normally (rev == root_noderev->predecessor_count), but here we use a more roundabout check that should only trigger on new instances - of the corruption, rather then trigger on each and every new commit + of the corruption, rather than trigger on each and every new commit to a repository that has triggered the bug somewhere in its root noderev's history. */ @@ -2836,6 +3080,9 @@ get_final_id(svn_fs_x__id_t *part, INITIAL_OFFSET is the offset of the proto-rev-file on entry to commit_body. + Collect the pair_cache_key_t of all directories written to the + committed cache in DIRECTORY_IDS. + If REPS_TO_CACHE is not NULL, append to it a copy (allocated in REPS_POOL) of each data rep that is new in this revision. @@ -2847,6 +3094,10 @@ get_final_id(svn_fs_x__id_t *part, node-revision. It is only controls additional sanity checking logic. + CHANGED_PATHS is the changed paths hash for the new revision. + The noderev-ids in it will be updated as soon as the respective + nodesrevs got their final IDs assigned. + Temporary allocations are also from SCRATCH_POOL. */ static svn_error_t * write_final_rev(svn_fs_x__id_t *new_id_p, @@ -2855,10 +3106,12 @@ write_final_rev(svn_fs_x__id_t *new_id_p, svn_fs_t *fs, const svn_fs_x__id_t *id, apr_off_t initial_offset, + apr_array_header_t *directory_ids, apr_array_header_t *reps_to_cache, apr_hash_t *reps_hash, apr_pool_t *reps_pool, svn_boolean_t at_root, + apr_hash_t *changed_paths, apr_pool_t *scratch_pool) { svn_fs_x__noderev_t *noderev; @@ -2899,24 +3152,44 @@ write_final_rev(svn_fs_x__id_t *new_id_p, svn_pool_clear(subpool); SVN_ERR(write_final_rev(&new_id, file, rev, fs, &dirent->id, - initial_offset, reps_to_cache, reps_hash, - reps_pool, FALSE, subpool)); - if ( svn_fs_x__id_used(&new_id) - && (svn_fs_x__get_revnum(new_id.change_set) == rev)) + initial_offset, directory_ids, + reps_to_cache, reps_hash, + reps_pool, FALSE, changed_paths, subpool)); + if (new_id.change_set == change_set) dirent->id = new_id; } if (noderev->data_rep && ! svn_fs_x__is_revision(noderev->data_rep->id.change_set)) { + svn_fs_x__pair_cache_key_t *key; + svn_fs_x__dir_data_t dir_data; + /* Write out the contents of this directory as a text rep. */ noderev->data_rep->id.change_set = change_set; SVN_ERR(write_container_delta_rep(noderev->data_rep, file, entries, write_directory_to_stream, - fs, txn_id, noderev, NULL, + fs, txn_id, noderev, NULL, FALSE, SVN_FS_X__ITEM_TYPE_DIR_REP, rev, scratch_pool)); + + /* Cache the new directory contents. Otherwise, subsequent reads + * or commits will likely have to reconstruct, verify and parse + * it again. */ + key = apr_array_push(directory_ids); + key->revision = noderev->data_rep->id.change_set; + key->second = noderev->data_rep->id.number; + + /* Store directory contents under the new revision number but mark + * it as "stale" by setting the file length to 0. Committed dirs + * will report -1, in-txn dirs will report > 0, so that this can + * never match. We reset that to -1 after the commit is complete. + */ + dir_data.entries = entries; + dir_data.txn_filesize = 0; + + SVN_ERR(svn_cache__set(ffd->dir_cache, key, &dir_data, subpool)); } } else @@ -2949,8 +3222,8 @@ write_final_rev(svn_fs_x__id_t *new_id_p, SVN_ERR(write_container_delta_rep(noderev->prop_rep, file, proplist, write_hash_to_stream, fs, txn_id, - noderev, reps_hash, item_type, rev, - scratch_pool)); + noderev, reps_hash, TRUE, item_type, + rev, scratch_pool)); } /* Convert our temporary ID into a permanent revision one. */ @@ -2961,7 +3234,7 @@ write_final_rev(svn_fs_x__id_t *new_id_p, if (noderev->copyroot_rev == SVN_INVALID_REVNUM) noderev->copyroot_rev = rev; - SVN_ERR(svn_fs_x__get_file_offset(&my_offset, file, scratch_pool)); + SVN_ERR(svn_io_file_get_offset(&my_offset, file, scratch_pool)); SVN_ERR(store_l2p_index_entry(fs, txn_id, my_offset, noderev->noderev_id.number, scratch_pool)); @@ -3020,7 +3293,7 @@ write_final_rev(svn_fs_x__id_t *new_id_p, noderev_id.change_set = SVN_FS_X__INVALID_CHANGE_SET; entry.offset = my_offset; - SVN_ERR(svn_fs_x__get_file_offset(&my_offset, file, scratch_pool)); + SVN_ERR(svn_io_file_get_offset(&my_offset, file, scratch_pool)); entry.size = my_offset - entry.offset; entry.type = SVN_FS_X__ITEM_TYPE_NODEREV; entry.item_count = 1; @@ -3054,7 +3327,7 @@ write_final_changed_path_info(apr_off_t *offset_p, svn_fs_x__id_t rev_item = {SVN_INVALID_REVNUM, SVN_FS_X__ITEM_INDEX_CHANGES}; - SVN_ERR(svn_fs_x__get_file_offset(&offset, file, scratch_pool)); + SVN_ERR(svn_io_file_get_offset(&offset, file, scratch_pool)); /* write to target file & calculate checksum */ stream = svn_checksum__wrap_write_stream_fnv1a_32x4(&entry.fnv1_checksum, @@ -3068,7 +3341,7 @@ write_final_changed_path_info(apr_off_t *offset_p, /* reference changes from the indexes */ entry.offset = offset; - SVN_ERR(svn_fs_x__get_file_offset(&offset, file, scratch_pool)); + SVN_ERR(svn_io_file_get_offset(&offset, file, scratch_pool)); entry.size = offset - entry.offset; entry.type = SVN_FS_X__ITEM_TYPE_CHANGES; entry.item_count = 1; @@ -3199,62 +3472,52 @@ verify_locks(svn_fs_t *fs, return SVN_NO_ERROR; } -/* Return in *PATH the path to a file containing the properties that - make up the final revision properties file. This involves setting - svn:date and removing any temporary properties associated with the - commit flags. */ +/* Based on the transaction properties of TXN, write the final revision + properties for REVISION into their final location. Return that location + in *PATH and schedule the necessary fsync calls in BATCH. This involves + setting svn:date and removing any temporary properties associated with + the commit flags. */ static svn_error_t * write_final_revprop(const char **path, svn_fs_txn_t *txn, - svn_fs_x__txn_id_t txn_id, - apr_pool_t *pool) + svn_revnum_t revision, + svn_fs_x__batch_fsync_t *batch, + apr_pool_t *result_pool, + apr_pool_t *scratch_pool) { - apr_hash_t *txnprops; - svn_boolean_t final_mods = FALSE; + apr_hash_t *props; svn_string_t date; svn_string_t *client_date; + apr_file_t *file; - SVN_ERR(svn_fs_x__txn_proplist(&txnprops, txn, pool)); + SVN_ERR(svn_fs_x__txn_proplist(&props, txn, scratch_pool)); /* Remove any temporary txn props representing 'flags'. */ - if (svn_hash_gets(txnprops, SVN_FS__PROP_TXN_CHECK_OOD)) - { - svn_hash_sets(txnprops, SVN_FS__PROP_TXN_CHECK_OOD, NULL); - final_mods = TRUE; - } + if (svn_hash_gets(props, SVN_FS__PROP_TXN_CHECK_OOD)) + svn_hash_sets(props, SVN_FS__PROP_TXN_CHECK_OOD, NULL); - if (svn_hash_gets(txnprops, SVN_FS__PROP_TXN_CHECK_LOCKS)) - { - svn_hash_sets(txnprops, SVN_FS__PROP_TXN_CHECK_LOCKS, NULL); - final_mods = TRUE; - } + if (svn_hash_gets(props, SVN_FS__PROP_TXN_CHECK_LOCKS)) + svn_hash_sets(props, SVN_FS__PROP_TXN_CHECK_LOCKS, NULL); - client_date = svn_hash_gets(txnprops, SVN_FS__PROP_TXN_CLIENT_DATE); + client_date = svn_hash_gets(props, SVN_FS__PROP_TXN_CLIENT_DATE); if (client_date) - { - svn_hash_sets(txnprops, SVN_FS__PROP_TXN_CLIENT_DATE, NULL); - final_mods = TRUE; - } + svn_hash_sets(props, SVN_FS__PROP_TXN_CLIENT_DATE, NULL); /* Update commit time to ensure that svn:date revprops remain ordered if requested. */ if (!client_date || strcmp(client_date->data, "1")) { - date.data = svn_time_to_cstring(apr_time_now(), pool); + date.data = svn_time_to_cstring(apr_time_now(), scratch_pool); date.len = strlen(date.data); - svn_hash_sets(txnprops, SVN_PROP_REVISION_DATE, &date); - final_mods = TRUE; + svn_hash_sets(props, SVN_PROP_REVISION_DATE, &date); } - if (final_mods) - { - SVN_ERR(set_txn_proplist(txn->fs, txn_id, txnprops, TRUE, pool)); - *path = svn_fs_x__path_txn_props_final(txn->fs, txn_id, pool); - } - else - { - *path = svn_fs_x__path_txn_props(txn->fs, txn_id, pool); - } + /* Create a file at the final revprops location. */ + *path = svn_fs_x__path_revprops(txn->fs, revision, result_pool); + SVN_ERR(svn_fs_x__batch_fsync_open_file(&file, batch, *path, scratch_pool)); + + /* Write the new contents to the final revprops file. */ + SVN_ERR(svn_fs_x__write_non_packed_revprops(file, props, scratch_pool)); return SVN_NO_ERROR; } @@ -3302,6 +3565,190 @@ svn_fs_x__add_index_data(svn_fs_t *fs, return SVN_NO_ERROR; } +/* Make sure that the shard folder for REVSION exists in FS. If we had to + create them, schedule their fsync in BATCH. Use SCRATCH_POOL for + temporary allocations. */ +static svn_error_t * +auto_create_shard(svn_fs_t *fs, + svn_revnum_t revision, + svn_fs_x__batch_fsync_t *batch, + apr_pool_t *scratch_pool) +{ + svn_fs_x__data_t *ffd = fs->fsap_data; + if (revision % ffd->max_files_per_dir == 0) + { + const char *new_dir = svn_fs_x__path_shard(fs, revision, scratch_pool); + svn_error_t *err = svn_io_dir_make(new_dir, APR_OS_DEFAULT, + scratch_pool); + + if (err && !APR_STATUS_IS_EEXIST(err->apr_err)) + return svn_error_trace(err); + svn_error_clear(err); + + SVN_ERR(svn_io_copy_perms(svn_dirent_join(fs->path, PATH_REVS_DIR, + scratch_pool), + new_dir, scratch_pool)); + SVN_ERR(svn_fs_x__batch_fsync_new_path(batch, new_dir, scratch_pool)); + } + + return SVN_NO_ERROR; +} + +/* Move the protype revision file of transaction TXN_ID in FS to the final + location for REVISION and return a handle to it in *FILE. Schedule any + fsyncs in BATCH and use SCRATCH_POOL for temporaries. + + Note that the lifetime of *FILE is determined by BATCH instead of + SCRATCH_POOL. It will be invalidated by either BATCH being cleaned up + itself of by running svn_fs_x__batch_fsync_run on it. + + This function will "destroy" the transaction by removing its prototype + revision file, so it can at most be called once per transaction. Also, + later attempts to modify this txn will fail due to get_writable_proto_rev + not finding the protorev file. Therefore, we will take out the lock for + it only until we move the file to its final location. + + If the prototype revision file is already locked, return error + SVN_ERR_FS_REP_BEING_WRITTEN. */ +static svn_error_t * +get_writable_final_rev(apr_file_t **file, + svn_fs_t *fs, + svn_fs_x__txn_id_t txn_id, + svn_revnum_t revision, + svn_fs_x__batch_fsync_t *batch, + apr_pool_t *scratch_pool) +{ + get_writable_proto_rev_baton_t baton; + apr_off_t end_offset = 0; + void *lockcookie; + + const char *proto_rev_filename + = svn_fs_x__path_txn_proto_rev(fs, txn_id, scratch_pool); + const char *final_rev_filename + = svn_fs_x__path_rev(fs, revision, scratch_pool); + + /* Acquire exclusive access to the proto-rev file. */ + baton.lockcookie = &lockcookie; + baton.txn_id = txn_id; + + SVN_ERR(with_txnlist_lock(fs, get_writable_proto_rev_body, &baton, + scratch_pool)); + + /* Move the proto-rev file to its final location as revision data file. + After that, we don't need to protect it anymore and can unlock it. */ + SVN_ERR(svn_error_compose_create(svn_io_file_rename2(proto_rev_filename, + final_rev_filename, + FALSE, + scratch_pool), + unlock_proto_rev(fs, txn_id, lockcookie, + scratch_pool))); + SVN_ERR(svn_fs_x__batch_fsync_new_path(batch, final_rev_filename, + scratch_pool)); + + /* Now open the prototype revision file and seek to the end. + Note that BATCH always seeks to position 0 before returning the file. */ + SVN_ERR(svn_fs_x__batch_fsync_open_file(file, batch, final_rev_filename, + scratch_pool)); + SVN_ERR(svn_io_file_seek(*file, APR_END, &end_offset, scratch_pool)); + + /* We don't want unused sections (such as leftovers from failed delta + stream) in our file. Detect and fix those cases by truncating the + protorev file. */ + SVN_ERR(auto_truncate_proto_rev(fs, *file, end_offset, txn_id, + scratch_pool)); + + return SVN_NO_ERROR; +} + +/* Write REVISION into FS' 'next' file and schedule necessary fsyncs in BATCH. + Use SCRATCH_POOL for temporary allocations. */ +static svn_error_t * +write_next_file(svn_fs_t *fs, + svn_revnum_t revision, + svn_fs_x__batch_fsync_t *batch, + apr_pool_t *scratch_pool) +{ + apr_file_t *file; + const char *path = svn_fs_x__path_next(fs, scratch_pool); + const char *perms_path = svn_fs_x__path_current(fs, scratch_pool); + char *buf; + + /* Create / open the 'next' file. */ + SVN_ERR(svn_fs_x__batch_fsync_open_file(&file, batch, path, scratch_pool)); + + /* Write its contents. */ + buf = apr_psprintf(scratch_pool, "%ld\n", revision); + SVN_ERR(svn_io_file_write_full(file, buf, strlen(buf), NULL, scratch_pool)); + + /* Adjust permissions. */ + SVN_ERR(svn_io_copy_perms(perms_path, path, scratch_pool)); + + return SVN_NO_ERROR; +} + +/* Bump the 'current' file in FS to NEW_REV. Schedule fsyncs in BATCH. + * Use SCRATCH_POOL for temporary allocations. */ +static svn_error_t * +bump_current(svn_fs_t *fs, + svn_revnum_t new_rev, + svn_fs_x__batch_fsync_t *batch, + apr_pool_t *scratch_pool) +{ + const char *current_filename; + + /* Write the 'next' file. */ + SVN_ERR(write_next_file(fs, new_rev, batch, scratch_pool)); + + /* Commit all changes to disk. */ + SVN_ERR(svn_fs_x__batch_fsync_run(batch, scratch_pool)); + + /* Make the revision visible to all processes and threads. */ + current_filename = svn_fs_x__path_current(fs, scratch_pool); + SVN_ERR(svn_fs_x__move_into_place(svn_fs_x__path_next(fs, scratch_pool), + current_filename, current_filename, + batch, scratch_pool)); + + /* Make the new revision permanently visible. */ + SVN_ERR(svn_fs_x__batch_fsync_run(batch, scratch_pool)); + + return SVN_NO_ERROR; +} + +/* Mark the directories cached in FS with the keys from DIRECTORY_IDS + * as "valid" now. Use SCRATCH_POOL for temporaries. */ +static svn_error_t * +promote_cached_directories(svn_fs_t *fs, + apr_array_header_t *directory_ids, + apr_pool_t *scratch_pool) +{ + svn_fs_x__data_t *ffd = fs->fsap_data; + apr_pool_t *iterpool; + int i; + + if (!ffd->dir_cache) + return SVN_NO_ERROR; + + iterpool = svn_pool_create(scratch_pool); + for (i = 0; i < directory_ids->nelts; ++i) + { + const svn_fs_x__pair_cache_key_t *key + = &APR_ARRAY_IDX(directory_ids, i, svn_fs_x__pair_cache_key_t); + + svn_pool_clear(iterpool); + + /* Currently, the entry for KEY - if it still exists - is marked + * as "stale" and would not be used. Mark it as current for in- + * revison data. */ + SVN_ERR(svn_cache__set_partial(ffd->dir_cache, key, + svn_fs_x__reset_txn_filesize, NULL, + iterpool)); + } + + svn_pool_destroy(iterpool); + + return SVN_NO_ERROR; +} + /* Baton used for commit_body below. */ typedef struct commit_baton_t { svn_revnum_t *new_rev_p; @@ -3321,15 +3768,22 @@ commit_body(void *baton, { commit_baton_t *cb = baton; svn_fs_x__data_t *ffd = cb->fs->fsap_data; - const char *old_rev_filename, *rev_filename, *proto_filename; - const char *revprop_filename, *final_revprop; + const char *old_rev_filename, *rev_filename; + const char *revprop_filename; svn_fs_x__id_t root_id, new_root_id; svn_revnum_t old_rev, new_rev; apr_file_t *proto_file; - void *proto_file_lockcookie; apr_off_t initial_offset, changed_path_offset; svn_fs_x__txn_id_t txn_id = svn_fs_x__txn_get_id(cb->txn); apr_hash_t *changed_paths; + svn_fs_x__batch_fsync_t *batch; + apr_array_header_t *directory_ids + = apr_array_make(scratch_pool, 4, sizeof(svn_fs_x__pair_cache_key_t)); + + /* We perform a sequence of (potentially) large allocations. + Keep the peak memory usage low by using a SUBPOOL and cleaning it + up frequently. */ + apr_pool_t *subpool = svn_pool_create(scratch_pool); /* Re-Read the current repository format. All our repo upgrade and config evaluation strategies are such that existing information in @@ -3338,17 +3792,12 @@ commit_body(void *baton, Although we don't recommend upgrading hot repositories, people may still do it and we must make sure to either handle them gracefully or to error out. - - Committing pre-format 3 txns will fail after upgrade to format 3+ - because the proto-rev cannot be found; no further action needed. - Upgrades from pre-f7 to f7+ means a potential change in addressing - mode for the final rev. We must be sure to detect that cause because - the failure would only manifest once the new revision got committed. */ - SVN_ERR(svn_fs_x__read_format_file(cb->fs, scratch_pool)); + SVN_ERR(svn_fs_x__read_format_file(cb->fs, subpool)); /* Get the current youngest revision. */ - SVN_ERR(svn_fs_x__youngest_rev(&old_rev, cb->fs, scratch_pool)); + SVN_ERR(svn_fs_x__youngest_rev(&old_rev, cb->fs, subpool)); + svn_pool_clear(subpool); /* Check to make sure this transaction is based off the most recent revision. */ @@ -3365,111 +3814,70 @@ commit_body(void *baton, previous svn_fs.h functions and svn_fs_commit_txn(), so we need to re-examine every changed-path in the txn and re-verify all discovered locks. */ - SVN_ERR(verify_locks(cb->fs, txn_id, changed_paths, scratch_pool)); + SVN_ERR(verify_locks(cb->fs, txn_id, changed_paths, subpool)); + svn_pool_clear(subpool); /* We are going to be one better than this puny old revision. */ new_rev = old_rev + 1; - /* Get a write handle on the proto revision file. */ - SVN_ERR(get_writable_proto_rev(&proto_file, &proto_file_lockcookie, - cb->fs, txn_id, scratch_pool)); - SVN_ERR(svn_fs_x__get_file_offset(&initial_offset, proto_file, - scratch_pool)); + /* Use this to force all data to be flushed to physical storage + (to the degree our environment will allow). */ + SVN_ERR(svn_fs_x__batch_fsync_create(&batch, ffd->flush_to_disk, + scratch_pool)); + + /* Set up the target directory. */ + SVN_ERR(auto_create_shard(cb->fs, new_rev, batch, subpool)); + + /* Get a write handle on the proto revision file. + + ### This "breaks" the transaction by removing the protorev file + ### but the revision is not yet complete. If this commit does + ### not complete for any reason the transaction will be lost. */ + SVN_ERR(get_writable_final_rev(&proto_file, cb->fs, txn_id, new_rev, + batch, subpool)); + SVN_ERR(svn_io_file_get_offset(&initial_offset, proto_file, subpool)); + svn_pool_clear(subpool); /* Write out all the node-revisions and directory contents. */ svn_fs_x__init_txn_root(&root_id, txn_id); SVN_ERR(write_final_rev(&new_root_id, proto_file, new_rev, cb->fs, &root_id, - initial_offset, cb->reps_to_cache, cb->reps_hash, - cb->reps_pool, TRUE, scratch_pool)); + initial_offset, directory_ids, cb->reps_to_cache, + cb->reps_hash, cb->reps_pool, TRUE, changed_paths, + subpool)); + svn_pool_clear(subpool); /* Write the changed-path information. */ SVN_ERR(write_final_changed_path_info(&changed_path_offset, proto_file, cb->fs, txn_id, changed_paths, - new_rev, scratch_pool)); + new_rev, subpool)); + svn_pool_clear(subpool); /* Append the index data to the rev file. */ SVN_ERR(svn_fs_x__add_index_data(cb->fs, proto_file, - svn_fs_x__path_l2p_proto_index(cb->fs, txn_id, scratch_pool), - svn_fs_x__path_p2l_proto_index(cb->fs, txn_id, scratch_pool), - new_rev, scratch_pool)); - - SVN_ERR(svn_io_file_flush_to_disk(proto_file, scratch_pool)); - SVN_ERR(svn_io_file_close(proto_file, scratch_pool)); - - /* We don't unlock the prototype revision file immediately to avoid a - race with another caller writing to the prototype revision file - before we commit it. */ - - /* Create the shard for the rev and revprop file, if we're sharding and - this is the first revision of a new shard. We don't care if this - fails because the shard already existed for some reason. */ - if (new_rev % ffd->max_files_per_dir == 0) - { - /* Create the revs shard. */ - { - const char *new_dir - = svn_fs_x__path_rev_shard(cb->fs, new_rev, scratch_pool); - svn_error_t *err = svn_io_dir_make(new_dir, APR_OS_DEFAULT, - scratch_pool); - if (err && !APR_STATUS_IS_EEXIST(err->apr_err)) - return svn_error_trace(err); - svn_error_clear(err); - SVN_ERR(svn_io_copy_perms(svn_dirent_join(cb->fs->path, - PATH_REVS_DIR, - scratch_pool), - new_dir, scratch_pool)); - } - - /* Create the revprops shard. */ - SVN_ERR_ASSERT(! svn_fs_x__is_packed_revprop(cb->fs, new_rev)); - { - const char *new_dir - = svn_fs_x__path_revprops_shard(cb->fs, new_rev, scratch_pool); - svn_error_t *err = svn_io_dir_make(new_dir, APR_OS_DEFAULT, - scratch_pool); - if (err && !APR_STATUS_IS_EEXIST(err->apr_err)) - return svn_error_trace(err); - svn_error_clear(err); - SVN_ERR(svn_io_copy_perms(svn_dirent_join(cb->fs->path, - PATH_REVPROPS_DIR, - scratch_pool), - new_dir, scratch_pool)); - } - } - - /* Move the finished rev file into place. + svn_fs_x__path_l2p_proto_index(cb->fs, txn_id, subpool), + svn_fs_x__path_p2l_proto_index(cb->fs, txn_id, subpool), + new_rev, subpool)); + svn_pool_clear(subpool); - ### This "breaks" the transaction by removing the protorev file - ### but the revision is not yet complete. If this commit does - ### not complete for any reason the transaction will be lost. */ - old_rev_filename = svn_fs_x__path_rev_absolute(cb->fs, old_rev, - scratch_pool); - - rev_filename = svn_fs_x__path_rev(cb->fs, new_rev, scratch_pool); - proto_filename = svn_fs_x__path_txn_proto_rev(cb->fs, txn_id, - scratch_pool); - SVN_ERR(svn_fs_x__move_into_place(proto_filename, rev_filename, - old_rev_filename, scratch_pool)); - - /* Now that we've moved the prototype revision file out of the way, - we can unlock it (since further attempts to write to the file - will fail as it no longer exists). We must do this so that we can - remove the transaction directory later. */ - SVN_ERR(unlock_proto_rev(cb->fs, txn_id, proto_file_lockcookie, - scratch_pool)); + /* Set the correct permissions. */ + old_rev_filename = svn_fs_x__path_rev_absolute(cb->fs, old_rev, subpool); + rev_filename = svn_fs_x__path_rev(cb->fs, new_rev, subpool); + SVN_ERR(svn_io_copy_perms(rev_filename, old_rev_filename, subpool)); /* Move the revprops file into place. */ SVN_ERR_ASSERT(! svn_fs_x__is_packed_revprop(cb->fs, new_rev)); - SVN_ERR(write_final_revprop(&revprop_filename, cb->txn, txn_id, - scratch_pool)); - final_revprop = svn_fs_x__path_revprops(cb->fs, new_rev, scratch_pool); - SVN_ERR(svn_fs_x__move_into_place(revprop_filename, final_revprop, - old_rev_filename, scratch_pool)); + SVN_ERR(write_final_revprop(&revprop_filename, cb->txn, new_rev, batch, + subpool, subpool)); + SVN_ERR(svn_io_copy_perms(revprop_filename, old_rev_filename, subpool)); + svn_pool_clear(subpool); - /* Update the 'current' file. */ + /* Verify contents (no-op outside DEBUG mode). */ + SVN_ERR(svn_io_file_flush(proto_file, subpool)); SVN_ERR(verify_as_revision_before_current_plus_plus(cb->fs, new_rev, - scratch_pool)); - SVN_ERR(svn_fs_x__write_current(cb->fs, new_rev, scratch_pool)); + subpool)); + + /* Bump 'current'. */ + SVN_ERR(bump_current(cb->fs, new_rev, batch, subpool)); /* At this point the new revision is committed and globally visible so let the caller know it succeeded by giving it the new revision @@ -3480,9 +3888,14 @@ commit_body(void *baton, ffd->youngest_rev_cache = new_rev; + /* Make the directory contents already cached for the new revision + * visible. */ + SVN_ERR(promote_cached_directories(cb->fs, directory_ids, subpool)); + /* Remove this transaction directory. */ - SVN_ERR(svn_fs_x__purge_txn(cb->fs, cb->txn->id, scratch_pool)); + SVN_ERR(svn_fs_x__purge_txn(cb->fs, cb->txn->id, subpool)); + svn_pool_destroy(subpool); return SVN_NO_ERROR; } @@ -3542,6 +3955,8 @@ svn_fs_x__commit(svn_revnum_t *new_rev_p, if (ffd->rep_sharing_allowed) { + svn_error_t *err; + SVN_ERR(svn_fs_x__open_rep_cache(fs, scratch_pool)); /* Write new entries to the rep-sharing database. @@ -3552,9 +3967,21 @@ svn_fs_x__commit(svn_revnum_t *new_rev_p, /* ### A commit that touches thousands of files will starve other (reader/writer) commits for the duration of the below call. Maybe write in batches? */ - SVN_SQLITE__WITH_TXN( - write_reps_to_cache(fs, cb.reps_to_cache, scratch_pool), - ffd->rep_cache_db); + SVN_ERR(svn_sqlite__begin_transaction(ffd->rep_cache_db)); + err = write_reps_to_cache(fs, cb.reps_to_cache, scratch_pool); + err = svn_sqlite__finish_transaction(ffd->rep_cache_db, err); + + if (svn_error_find_cause(err, SVN_ERR_SQLITE_ROLLBACK_FAILED)) + { + /* Failed rollback means that our db connection is unusable, and + the only thing we can do is close it. The connection will be + reopened during the next operation with rep-cache.db. */ + return svn_error_trace( + svn_error_compose_create(err, + svn_fs_x__close_rep_cache(fs))); + } + else if (err) + return svn_error_trace(err); } return SVN_NO_ERROR; @@ -3650,10 +4077,8 @@ svn_fs_x__txn_proplist(apr_hash_t **table_p, svn_fs_txn_t *txn, apr_pool_t *pool) { - apr_hash_t *proplist = apr_hash_make(pool); - SVN_ERR(get_txn_proplist(proplist, txn->fs, svn_fs_x__txn_get_id(txn), - pool)); - *table_p = proplist; + SVN_ERR(get_txn_proplist(table_p, txn->fs, svn_fs_x__txn_get_id(txn), + pool, pool)); return SVN_NO_ERROR; } @@ -3776,7 +4201,7 @@ svn_fs_x__begin_txn(svn_fs_txn_t **txn_p, svn_string_create("0", scratch_pool)); ftd = (*txn_p)->fsap_data; - SVN_ERR(set_txn_proplist(fs, ftd->txn_id, props, FALSE, scratch_pool)); + SVN_ERR(set_txn_proplist(fs, ftd->txn_id, props, scratch_pool)); return SVN_NO_ERROR; } diff --git a/subversion/libsvn_fs_x/transaction.h b/subversion/libsvn_fs_x/transaction.h index 490f716d4333..9ed655bc8adf 100644 --- a/subversion/libsvn_fs_x/transaction.h +++ b/subversion/libsvn_fs_x/transaction.h @@ -20,8 +20,8 @@ * ==================================================================== */ -#ifndef SVN_LIBSVN_FS__TRANSACTION_H -#define SVN_LIBSVN_FS__TRANSACTION_H +#ifndef SVN_LIBSVN_FS_X_TRANSACTION_H +#define SVN_LIBSVN_FS_X_TRANSACTION_H #include "fs.h" @@ -64,8 +64,8 @@ svn_fs_x__with_txn_current_lock(svn_fs_t *fs, call BODY with BATON and that subpool, destroy the subpool (releasing the locks) and return what BODY returned. - This combines svn_fs_fs__with_write_lock, svn_fs_fs__with_pack_lock, - and svn_fs_fs__with_txn_current_lock, ensuring correct lock ordering. */ + This combines svn_fs_x__with_write_lock, svn_fs_x__with_pack_lock, + and svn_fs_x__with_txn_current_lock, ensuring correct lock ordering. */ svn_error_t * svn_fs_x__with_all_locks(svn_fs_t *fs, svn_error_t *(*body)(void *baton, @@ -128,7 +128,7 @@ svn_fs_x__reserve_copy_id(svn_fs_x__id_t *copy_id_p, /* Create an entirely new mutable node in the filesystem FS, whose node-revision is NODEREV. COPY_ID is the copy_id to use in the node revision ID. TXN_ID is the Subversion transaction under - which this occurs. */ + which this occurs. Use SCRATCH_POOL for temporary allocations. */ svn_error_t * svn_fs_x__create_node(svn_fs_t *fs, svn_fs_x__noderev_t *noderev, @@ -167,9 +167,9 @@ svn_fs_x__set_entry(svn_fs_t *fs, apr_pool_t *scratch_pool); /* Add a change to the changes record for filesystem FS in transaction - TXN_ID. Mark path PATH, having noderev-id ID, as changed according to - the type in CHANGE_KIND. If the text representation was changed set - TEXT_MOD to TRUE, and likewise for PROP_MOD as well as MERGEINFO_MOD. + TXN_ID. Mark path PATH as changed according to the type in + CHANGE_KIND. If the text representation was changed set TEXT_MOD + to TRUE, and likewise for PROP_MOD as well as MERGEINFO_MOD. If this change was the result of a copy, set COPYFROM_REV and COPYFROM_PATH to the revision and path of the copy source, otherwise they should be set to SVN_INVALID_REVNUM and NULL. Perform any @@ -178,7 +178,6 @@ svn_error_t * svn_fs_x__add_change(svn_fs_t *fs, svn_fs_x__txn_id_t txn_id, const char *path, - const svn_fs_x__id_t *id, svn_fs_path_change_kind_t change_kind, svn_boolean_t text_mod, svn_boolean_t prop_mod, diff --git a/subversion/libsvn_fs_x/tree.c b/subversion/libsvn_fs_x/tree.c index ce247658d30d..36374afc1cce 100644 --- a/subversion/libsvn_fs_x/tree.c +++ b/subversion/libsvn_fs_x/tree.c @@ -53,6 +53,7 @@ #include "fs.h" #include "dag.h" +#include "dag_cache.h" #include "lock.h" #include "tree.h" #include "fs_x.h" @@ -90,19 +91,8 @@ typedef struct fs_txn_root_data_t { /* TXN_ID value from the main struct but as a struct instead of a string */ svn_fs_x__txn_id_t txn_id; - - /* Cache of txn DAG nodes (without their nested noderevs, because - * it's mutable). Same keys/values as ffd->rev_node_cache. */ - svn_cache__t *txn_node_cache; } fs_txn_root_data_t; -/* Declared here to resolve the circular dependencies. */ -static svn_error_t * -get_dag(dag_node_t **dag_node_p, - svn_fs_root_t *root, - const char *path, - apr_pool_t *pool); - static svn_fs_root_t * make_revision_root(svn_fs_t *fs, svn_revnum_t rev, @@ -124,391 +114,6 @@ x_closest_copy(svn_fs_root_t **root_p, apr_pool_t *pool); -/*** Node Caching ***/ - -/* 1st level cache */ - -/* An entry in the first-level cache. REVISION and PATH form the key that - will ultimately be matched. - */ -typedef struct cache_entry_t -{ - /* hash value derived from PATH, REVISION. - Used to short-circuit failed lookups. */ - apr_uint32_t hash_value; - - /* revision to which the NODE belongs */ - svn_revnum_t revision; - - /* path of the NODE */ - char *path; - - /* cached value of strlen(PATH). */ - apr_size_t path_len; - - /* the node allocated in the cache's pool. NULL for empty entries. */ - dag_node_t *node; -} cache_entry_t; - -/* Number of entries in the cache. Keep this low to keep pressure on the - CPU caches low as well. A binary value is most efficient. If we walk - a directory tree, we want enough entries to store nodes for all files - without overwriting the nodes for the parent folder. That way, there - will be no unnecessary misses (except for a few random ones caused by - hash collision). - - The actual number of instances may be higher but entries that got - overwritten are no longer visible. - */ -enum { BUCKET_COUNT = 256 }; - -/* The actual cache structure. All nodes will be allocated in POOL. - When the number of INSERTIONS (i.e. objects created form that pool) - exceeds a certain threshold, the pool will be cleared and the cache - with it. - */ -struct svn_fs_x__dag_cache_t -{ - /* fixed number of (possibly empty) cache entries */ - cache_entry_t buckets[BUCKET_COUNT]; - - /* pool used for all node allocation */ - apr_pool_t *pool; - - /* number of entries created from POOL since the last cleanup */ - apr_size_t insertions; - - /* Property lookups etc. have a very high locality (75% re-hit). - Thus, remember the last hit location for optimistic lookup. */ - apr_size_t last_hit; - - /* Position of the last bucket hit that actually had a DAG node in it. - LAST_HIT may refer to a bucket that matches path@rev but has not - its NODE element set, yet. - This value is a mere hint for optimistic lookup and any value is - valid (as long as it is < BUCKET_COUNT). */ - apr_size_t last_non_empty; -}; - -svn_fs_x__dag_cache_t* -svn_fs_x__create_dag_cache(apr_pool_t *result_pool) -{ - svn_fs_x__dag_cache_t *result = apr_pcalloc(result_pool, sizeof(*result)); - result->pool = svn_pool_create(result_pool); - - return result; -} - -/* Clears the CACHE at regular intervals (destroying all cached nodes) - */ -static void -auto_clear_dag_cache(svn_fs_x__dag_cache_t* cache) -{ - if (cache->insertions > BUCKET_COUNT) - { - svn_pool_clear(cache->pool); - - memset(cache->buckets, 0, sizeof(cache->buckets)); - cache->insertions = 0; - } -} - -/* For the given REVISION and PATH, return the respective entry in CACHE. - If the entry is empty, its NODE member will be NULL and the caller - may then set it to the corresponding DAG node allocated in CACHE->POOL. - */ -static cache_entry_t * -cache_lookup( svn_fs_x__dag_cache_t *cache - , svn_revnum_t revision - , const char *path) -{ - apr_size_t i, bucket_index; - apr_size_t path_len = strlen(path); - apr_uint32_t hash_value = (apr_uint32_t)revision; - -#if SVN_UNALIGNED_ACCESS_IS_OK - /* "randomizing" / distributing factor used in our hash function */ - const apr_uint32_t factor = 0xd1f3da69; -#endif - - /* optimistic lookup: hit the same bucket again? */ - cache_entry_t *result = &cache->buckets[cache->last_hit]; - if ( (result->revision == revision) - && (result->path_len == path_len) - && !memcmp(result->path, path, path_len)) - { - /* Remember the position of the last node we found in this cache. */ - if (result->node) - cache->last_non_empty = cache->last_hit; - - return result; - } - - /* need to do a full lookup. Calculate the hash value - (HASH_VALUE has been initialized to REVISION). */ - i = 0; -#if SVN_UNALIGNED_ACCESS_IS_OK - /* We relax the dependency chain between iterations by processing - two chunks from the input per hash_value self-multiplication. - The HASH_VALUE update latency is now 1 MUL latency + 1 ADD latency - per 2 chunks instead of 1 chunk. - */ - for (; i + 8 <= path_len; i += 8) - hash_value = hash_value * factor * factor - + ( *(const apr_uint32_t*)(path + i) * factor - + *(const apr_uint32_t*)(path + i + 4)); -#endif - - for (; i < path_len; ++i) - /* Help GCC to minimize the HASH_VALUE update latency by splitting the - MUL 33 of the naive implementation: h = h * 33 + path[i]. This - shortens the dependency chain from 1 shift + 2 ADDs to 1 shift + 1 ADD. - */ - hash_value = hash_value * 32 + (hash_value + (unsigned char)path[i]); - - bucket_index = hash_value + (hash_value >> 16); - bucket_index = (bucket_index + (bucket_index >> 8)) % BUCKET_COUNT; - - /* access the corresponding bucket and remember its location */ - result = &cache->buckets[bucket_index]; - cache->last_hit = bucket_index; - - /* if it is *NOT* a match, clear the bucket, expect the caller to fill - in the node and count it as an insertion */ - if ( (result->hash_value != hash_value) - || (result->revision != revision) - || (result->path_len != path_len) - || memcmp(result->path, path, path_len)) - { - result->hash_value = hash_value; - result->revision = revision; - if (result->path_len < path_len) - result->path = apr_palloc(cache->pool, path_len + 1); - result->path_len = path_len; - memcpy(result->path, path, path_len + 1); - - result->node = NULL; - - cache->insertions++; - } - else if (result->node) - { - /* This bucket is valid & has a suitable DAG node in it. - Remember its location. */ - cache->last_non_empty = bucket_index; - } - - return result; -} - -/* Optimistic lookup using the last seen non-empty location in CACHE. - Return the node of that entry, if it is still in use and matches PATH. - Return NULL otherwise. Since the caller usually already knows the path - length, provide it in PATH_LEN. */ -static dag_node_t * -cache_lookup_last_path(svn_fs_x__dag_cache_t *cache, - const char *path, - apr_size_t path_len) -{ - cache_entry_t *result = &cache->buckets[cache->last_non_empty]; - assert(strlen(path) == path_len); - - if ( result->node - && (result->path_len == path_len) - && !memcmp(result->path, path, path_len)) - { - return result->node; - } - - return NULL; -} - -/* 2nd level cache */ - -/* Find and return the DAG node cache for ROOT and the key that - should be used for PATH. - - RESULT_POOL will only be used for allocating a new keys if necessary. */ -static void -locate_cache(svn_cache__t **cache, - const char **key, - svn_fs_root_t *root, - const char *path, - apr_pool_t *result_pool) -{ - if (root->is_txn_root) - { - fs_txn_root_data_t *frd = root->fsap_data; - - if (cache) - *cache = frd->txn_node_cache; - if (key && path) - *key = path; - } - else - { - svn_fs_x__data_t *ffd = root->fs->fsap_data; - - if (cache) - *cache = ffd->rev_node_cache; - if (key && path) - *key = svn_fs_x__combine_number_and_string(root->rev, path, - result_pool); - } -} - -/* Return NODE for PATH from ROOT's node cache, or NULL if the node - isn't cached; read it from the FS. *NODE remains valid until either - POOL or the FS gets cleared or destroyed (whichever comes first). - */ -static svn_error_t * -dag_node_cache_get(dag_node_t **node_p, - svn_fs_root_t *root, - const char *path, - apr_pool_t *pool) -{ - svn_boolean_t found; - dag_node_t *node = NULL; - svn_cache__t *cache; - const char *key; - - SVN_ERR_ASSERT(*path == '/'); - - if (!root->is_txn_root) - { - /* immutable DAG node. use the global caches for it */ - - svn_fs_x__data_t *ffd = root->fs->fsap_data; - cache_entry_t *bucket; - - auto_clear_dag_cache(ffd->dag_node_cache); - bucket = cache_lookup(ffd->dag_node_cache, root->rev, path); - if (bucket->node == NULL) - { - locate_cache(&cache, &key, root, path, pool); - SVN_ERR(svn_cache__get((void **)&node, &found, cache, key, - ffd->dag_node_cache->pool)); - if (found && node) - { - /* Patch up the FS, since this might have come from an old FS - * object. */ - svn_fs_x__dag_set_fs(node, root->fs); - bucket->node = node; - } - } - else - { - node = bucket->node; - } - } - else - { - /* DAG is mutable / may become invalid. Use the TXN-local cache */ - - locate_cache(&cache, &key, root, path, pool); - - SVN_ERR(svn_cache__get((void **) &node, &found, cache, key, pool)); - if (found && node) - { - /* Patch up the FS, since this might have come from an old FS - * object. */ - svn_fs_x__dag_set_fs(node, root->fs); - } - } - - *node_p = node; - - return SVN_NO_ERROR; -} - - -/* Add the NODE for PATH to ROOT's node cache. */ -static svn_error_t * -dag_node_cache_set(svn_fs_root_t *root, - const char *path, - dag_node_t *node, - apr_pool_t *scratch_pool) -{ - svn_cache__t *cache; - const char *key; - - SVN_ERR_ASSERT(*path == '/'); - - /* Do *not* attempt to dup and put the node into L1. - * dup() is twice as expensive as an L2 lookup (which will set also L1). - */ - locate_cache(&cache, &key, root, path, scratch_pool); - - return svn_cache__set(cache, key, node, scratch_pool); -} - - -/* Baton for find_descendants_in_cache. */ -typedef struct fdic_baton_t -{ - const char *path; - apr_array_header_t *list; - apr_pool_t *pool; -} fdic_baton_t; - -/* If the given item is a descendant of BATON->PATH, push - * it onto BATON->LIST (copying into BATON->POOL). Implements - * the svn_iter_apr_hash_cb_t prototype. */ -static svn_error_t * -find_descendants_in_cache(void *baton, - const void *key, - apr_ssize_t klen, - void *val, - apr_pool_t *pool) -{ - fdic_baton_t *b = baton; - const char *item_path = key; - - if (svn_fspath__skip_ancestor(b->path, item_path)) - APR_ARRAY_PUSH(b->list, const char *) = apr_pstrdup(b->pool, item_path); - - return SVN_NO_ERROR; -} - -/* Invalidate cache entries for PATH and any of its children. This - should *only* be called on a transaction root! */ -static svn_error_t * -dag_node_cache_invalidate(svn_fs_root_t *root, - const char *path, - apr_pool_t *scratch_pool) -{ - fdic_baton_t b; - svn_cache__t *cache; - apr_pool_t *iterpool; - int i; - - b.path = path; - b.pool = svn_pool_create(scratch_pool); - b.list = apr_array_make(b.pool, 1, sizeof(const char *)); - - SVN_ERR_ASSERT(root->is_txn_root); - locate_cache(&cache, NULL, root, NULL, b.pool); - - - SVN_ERR(svn_cache__iter(NULL, cache, find_descendants_in_cache, - &b, b.pool)); - - iterpool = svn_pool_create(b.pool); - - for (i = 0; i < b.list->nelts; i++) - { - const char *descendant = APR_ARRAY_IDX(b.list, i, const char *); - svn_pool_clear(iterpool); - SVN_ERR(svn_cache__set(cache, descendant, NULL, iterpool)); - } - - svn_pool_destroy(iterpool); - svn_pool_destroy(b.pool); - return SVN_NO_ERROR; -} - - - /* Creating transaction and revision root nodes. */ svn_error_t * @@ -554,8 +159,8 @@ svn_fs_x__revision_root(svn_fs_root_t **root_p, /* Getting dag nodes for roots. */ /* Return the transaction ID to a given transaction ROOT. */ -static svn_fs_x__txn_id_t -root_txn_id(svn_fs_root_t *root) +svn_fs_x__txn_id_t +svn_fs_x__root_txn_id(svn_fs_root_t *root) { fs_txn_root_data_t *frd = root->fsap_data; assert(root->is_txn_root); @@ -563,105 +168,32 @@ root_txn_id(svn_fs_root_t *root) return frd->txn_id; } -/* Set *NODE_P to a freshly opened dag node referring to the root - directory of ROOT, allocating from RESULT_POOL. Use SCRATCH_POOL - for temporary allocations. */ -static svn_error_t * -root_node(dag_node_t **node_p, - svn_fs_root_t *root, - apr_pool_t *result_pool, - apr_pool_t *scratch_pool) +/* Return the change set to a given ROOT. */ +svn_fs_x__change_set_t +svn_fs_x__root_change_set(svn_fs_root_t *root) { if (root->is_txn_root) - { - /* It's a transaction root. Open a fresh copy. */ - return svn_fs_x__dag_txn_root(node_p, root->fs, root_txn_id(root), - result_pool, scratch_pool); - } - else - { - /* It's a revision root, so we already have its root directory - opened. */ - return svn_fs_x__dag_revision_root(node_p, root->fs, root->rev, - result_pool, scratch_pool); - } -} - + return svn_fs_x__change_set_by_txn(svn_fs_x__root_txn_id(root)); -/* Set *NODE_P to a mutable root directory for ROOT, cloning if - necessary, allocating in RESULT_POOL. ROOT must be a transaction root. - Use ERROR_PATH in error messages. Use SCRATCH_POOL for temporaries.*/ -static svn_error_t * -mutable_root_node(dag_node_t **node_p, - svn_fs_root_t *root, - const char *error_path, - apr_pool_t *result_pool, - apr_pool_t *scratch_pool) -{ - if (root->is_txn_root) - { - /* It's a transaction root. Open a fresh copy. */ - return svn_fs_x__dag_txn_root(node_p, root->fs, root_txn_id(root), - result_pool, scratch_pool); - } - else - /* If it's not a transaction root, we can't change its contents. */ - return SVN_FS__ERR_NOT_MUTABLE(root->fs, root->rev, error_path); + return svn_fs_x__change_set_by_rev(root->rev); } + /* Traversing directory paths. */ -typedef enum copy_id_inherit_t -{ - copy_id_inherit_unknown = 0, - copy_id_inherit_self, - copy_id_inherit_parent, - copy_id_inherit_new - -} copy_id_inherit_t; - -/* A linked list representing the path from a node up to a root - directory. We use this for cloning, and for operations that need - to deal with both a node and its parent directory. For example, a - `delete' operation needs to know that the node actually exists, but - also needs to change the parent directory. */ -typedef struct parent_path_t -{ - - /* A node along the path. This could be the final node, one of its - parents, or the root. Every parent path ends with an element for - the root directory. */ - dag_node_t *node; - - /* The name NODE has in its parent directory. This is zero for the - root directory, which (obviously) has no name in its parent. */ - char *entry; - - /* The parent of NODE, or zero if NODE is the root directory. */ - struct parent_path_t *parent; - - /* The copy ID inheritance style. */ - copy_id_inherit_t copy_inherit; - - /* If copy ID inheritance style is copy_id_inherit_new, this is the - path which should be implicitly copied; otherwise, this is NULL. */ - const char *copy_src_path; - -} parent_path_t; - -/* Return a text string describing the absolute path of parent_path - PARENT_PATH. It will be allocated in POOL. */ +/* Return a text string describing the absolute path of parent path + DAG_PATH. It will be allocated in POOL. */ static const char * -parent_path_path(parent_path_t *parent_path, +parent_path_path(svn_fs_x__dag_path_t *dag_path, apr_pool_t *pool) { const char *path_so_far = "/"; - if (parent_path->parent) - path_so_far = parent_path_path(parent_path->parent, pool); - return parent_path->entry - ? svn_fspath__join(path_so_far, parent_path->entry, pool) + if (dag_path->parent) + path_so_far = parent_path_path(dag_path->parent, pool); + return dag_path->entry + ? svn_fspath__join(path_so_far, dag_path->entry, pool) : path_so_far; } @@ -669,12 +201,12 @@ parent_path_path(parent_path_t *parent_path, /* Return the FS path for the parent path chain object CHILD relative to its ANCESTOR in the same chain, allocated in POOL. */ static const char * -parent_path_relpath(parent_path_t *child, - parent_path_t *ancestor, +parent_path_relpath(svn_fs_x__dag_path_t *child, + svn_fs_x__dag_path_t *ancestor, apr_pool_t *pool) { const char *path_so_far = ""; - parent_path_t *this_node = child; + svn_fs_x__dag_path_t *this_node = child; while (this_node != ancestor) { assert(this_node != NULL); @@ -686,583 +218,22 @@ parent_path_relpath(parent_path_t *child, -/* Choose a copy ID inheritance method *INHERIT_P to be used in the - event that immutable node CHILD in FS needs to be made mutable. If - the inheritance method is copy_id_inherit_new, also return a - *COPY_SRC_PATH on which to base the new copy ID (else return NULL - for that path). CHILD must have a parent (it cannot be the root - node). Allocations are taken from POOL. */ -static svn_error_t * -get_copy_inheritance(copy_id_inherit_t *inherit_p, - const char **copy_src_path, - svn_fs_t *fs, - parent_path_t *child, - apr_pool_t *pool) -{ - svn_fs_x__id_t child_copy_id, parent_copy_id; - svn_boolean_t related; - const char *id_path = NULL; - svn_fs_root_t *copyroot_root; - dag_node_t *copyroot_node; - svn_revnum_t copyroot_rev; - const char *copyroot_path; - - SVN_ERR_ASSERT(child && child->parent); - - /* Initialize some convenience variables. */ - SVN_ERR(svn_fs_x__dag_get_copy_id(&child_copy_id, child->node)); - SVN_ERR(svn_fs_x__dag_get_copy_id(&parent_copy_id, child->parent->node)); - - /* If this child is already mutable, we have nothing to do. */ - if (svn_fs_x__dag_check_mutable(child->node)) - { - *inherit_p = copy_id_inherit_self; - *copy_src_path = NULL; - return SVN_NO_ERROR; - } - - /* From this point on, we'll assume that the child will just take - its copy ID from its parent. */ - *inherit_p = copy_id_inherit_parent; - *copy_src_path = NULL; - - /* Special case: if the child's copy ID is '0', use the parent's - copy ID. */ - if (svn_fs_x__id_is_root(&child_copy_id)) - return SVN_NO_ERROR; - - /* Compare the copy IDs of the child and its parent. If they are - the same, then the child is already on the same branch as the - parent, and should use the same mutability copy ID that the - parent will use. */ - if (svn_fs_x__id_eq(&child_copy_id, &parent_copy_id)) - return SVN_NO_ERROR; - - /* If the child is on the same branch that the parent is on, the - child should just use the same copy ID that the parent would use. - Else, the child needs to generate a new copy ID to use should it - need to be made mutable. We will claim that child is on the same - branch as its parent if the child itself is not a branch point, - or if it is a branch point that we are accessing via its original - copy destination path. */ - SVN_ERR(svn_fs_x__dag_get_copyroot(©root_rev, ©root_path, - child->node)); - SVN_ERR(svn_fs_x__revision_root(©root_root, fs, copyroot_rev, pool)); - SVN_ERR(get_dag(©root_node, copyroot_root, copyroot_path, pool)); - - SVN_ERR(svn_fs_x__dag_related_node(&related, copyroot_node, child->node)); - if (!related) - return SVN_NO_ERROR; - - /* Determine if we are looking at the child via its original path or - as a subtree item of a copied tree. */ - id_path = svn_fs_x__dag_get_created_path(child->node); - if (strcmp(id_path, parent_path_path(child, pool)) == 0) - { - *inherit_p = copy_id_inherit_self; - return SVN_NO_ERROR; - } - - /* We are pretty sure that the child node is an unedited nested - branched node. When it needs to be made mutable, it should claim - a new copy ID. */ - *inherit_p = copy_id_inherit_new; - *copy_src_path = id_path; - return SVN_NO_ERROR; -} - -/* Allocate a new parent_path_t node from RESULT_POOL, referring to NODE, - ENTRY, PARENT, and COPY_ID. */ -static parent_path_t * -make_parent_path(dag_node_t *node, - char *entry, - parent_path_t *parent, - apr_pool_t *result_pool) -{ - parent_path_t *parent_path = apr_pcalloc(result_pool, sizeof(*parent_path)); - if (node) - parent_path->node = svn_fs_x__dag_copy_into_pool(node, result_pool); - parent_path->entry = entry; - parent_path->parent = parent; - parent_path->copy_inherit = copy_id_inherit_unknown; - parent_path->copy_src_path = NULL; - return parent_path; -} - - -/* Flags for open_path. */ -typedef enum open_path_flags_t { - - /* The last component of the PATH need not exist. (All parent - directories must exist, as usual.) If the last component doesn't - exist, simply leave the `node' member of the bottom parent_path - component zero. */ - open_path_last_optional = 1, - - /* When this flag is set, don't bother to lookup the DAG node in - our caches because we already tried this. Ignoring this flag - has no functional impact. */ - open_path_uncached = 2, - - /* The caller does not care about the parent node chain but only - the final DAG node. */ - open_path_node_only = 4, - - /* The caller wants a NULL path object instead of an error if the - path cannot be found. */ - open_path_allow_null = 8 -} open_path_flags_t; - -/* Try a short-cut for the open_path() function using the last node accessed. - * If that ROOT is that nodes's "created rev" and PATH of PATH_LEN chars is - * its "created path", return the node in *NODE_P. Set it to NULL otherwise. - * - * This function is used to support ra_serf-style access patterns where we - * are first asked for path@rev and then for path@c_rev of the same node. - * The shortcut works by ignoring the "rev" part of the cache key and then - * checking whether we got lucky. Lookup and verification are both quick - * plus there are many early outs for common types of mismatch. - */ -static svn_error_t * -try_match_last_node(dag_node_t **node_p, - svn_fs_root_t *root, - const char *path, - apr_size_t path_len, - apr_pool_t *scratch_pool) -{ - svn_fs_x__data_t *ffd = root->fs->fsap_data; - - /* Optimistic lookup: if the last node returned from the cache applied to - the same PATH, return it in NODE. */ - dag_node_t *node - = cache_lookup_last_path(ffd->dag_node_cache, path, path_len); - - /* Did we get a bucket with a committed node? */ - if (node && !svn_fs_x__dag_check_mutable(node)) - { - /* Get the path&rev pair at which this node was created. - This is repository location for which this node is _known_ to be - the right lookup result irrespective of how we found it. */ - const char *created_path - = svn_fs_x__dag_get_created_path(node); - svn_revnum_t revision = svn_fs_x__dag_get_revision(node); - - /* Is it an exact match? */ - if (revision == root->rev && strcmp(created_path, path) == 0) - { - /* Cache it under its full path@rev access path. */ - SVN_ERR(dag_node_cache_set(root, path, node, scratch_pool)); - - *node_p = node; - return SVN_NO_ERROR; - } - } - - *node_p = NULL; - return SVN_NO_ERROR; -} - - -/* Open the node identified by PATH in ROOT, allocating in POOL. Set - *PARENT_PATH_P to a path from the node up to ROOT. The resulting - **PARENT_PATH_P value is guaranteed to contain at least one - *element, for the root directory. PATH must be in canonical form. - - If resulting *PARENT_PATH_P will eventually be made mutable and - modified, or if copy ID inheritance information is otherwise needed, - IS_TXN_PATH must be set. If IS_TXN_PATH is FALSE, no copy ID - inheritance information will be calculated for the *PARENT_PATH_P chain. - - If FLAGS & open_path_last_optional is zero, return the error - SVN_ERR_FS_NOT_FOUND if the node PATH refers to does not exist. If - non-zero, require all the parent directories to exist as normal, - but if the final path component doesn't exist, simply return a path - whose bottom `node' member is zero. This option is useful for - callers that create new nodes --- we find the parent directory for - them, and tell them whether the entry exists already. - - The remaining bits in FLAGS are hints that allow this function - to take shortcuts based on knowledge that the caller provides, - such as the caller is not actually being interested in PARENT_PATH_P, - but only in (*PARENT_PATH_P)->NODE. - - NOTE: Public interfaces which only *read* from the filesystem - should not call this function directly, but should instead use - get_dag(). -*/ -static svn_error_t * -open_path(parent_path_t **parent_path_p, - svn_fs_root_t *root, - const char *path, - int flags, - svn_boolean_t is_txn_path, - apr_pool_t *pool) -{ - svn_fs_t *fs = root->fs; - dag_node_t *here = NULL; /* The directory we're currently looking at. */ - parent_path_t *parent_path; /* The path from HERE up to the root. */ - const char *rest = NULL; /* The portion of PATH we haven't traversed yet. */ - apr_pool_t *iterpool = svn_pool_create(pool); - - /* path to the currently processed entry without trailing '/'. - We will reuse this across iterations by simply putting a NUL terminator - at the respective position and replacing that with a '/' in the next - iteration. This is correct as we assert() PATH to be canonical. */ - svn_stringbuf_t *path_so_far = svn_stringbuf_create(path, pool); - apr_size_t path_len = path_so_far->len; - - /* Callers often traverse the DAG in some path-based order or along the - history segments. That allows us to try a few guesses about where to - find the next item. This is only useful if the caller didn't request - the full parent chain. */ - assert(svn_fs__is_canonical_abspath(path)); - path_so_far->len = 0; /* "" */ - if (flags & open_path_node_only) - { - const char *directory; - - /* First attempt: Assume that we access the DAG for the same path as - in the last lookup but for a different revision that happens to be - the last revision that touched the respective node. This is a - common pattern when e.g. checking out over ra_serf. Note that this - will only work for committed data as the revision info for nodes in - txns is bogus. - - This shortcut is quick and will exit this function upon success. - So, try it first. */ - if (!root->is_txn_root) - { - dag_node_t *node; - SVN_ERR(try_match_last_node(&node, root, path, path_len, iterpool)); - - /* Did the shortcut work? */ - if (node) - { - /* Construct and return the result. */ - svn_pool_destroy(iterpool); - - parent_path = make_parent_path(node, 0, 0, pool); - parent_path->copy_inherit = copy_id_inherit_self; - *parent_path_p = parent_path; - - return SVN_NO_ERROR; - } - } - - /* Second attempt: Try starting the lookup immediately at the parent - node. We will often have recently accessed either a sibling or - said parent DIRECTORY itself for the same revision. */ - directory = svn_dirent_dirname(path, pool); - if (directory[1] != 0) /* root nodes are covered anyway */ - { - SVN_ERR(dag_node_cache_get(&here, root, directory, pool)); - - /* Did the shortcut work? */ - if (here) - { - apr_size_t dirname_len = strlen(directory); - path_so_far->len = dirname_len; - rest = path + dirname_len + 1; - } - } - } - - /* did the shortcut work? */ - if (!here) - { - /* Make a parent_path item for the root node, using its own current - copy id. */ - SVN_ERR(root_node(&here, root, pool, iterpool)); - rest = path + 1; /* skip the leading '/', it saves in iteration */ - } - - path_so_far->data[path_so_far->len] = '\0'; - parent_path = make_parent_path(here, 0, 0, pool); - parent_path->copy_inherit = copy_id_inherit_self; - - /* Whenever we are at the top of this loop: - - HERE is our current directory, - - ID is the node revision ID of HERE, - - REST is the path we're going to find in HERE, and - - PARENT_PATH includes HERE and all its parents. */ - for (;;) - { - const char *next; - char *entry; - dag_node_t *child; - - svn_pool_clear(iterpool); - - /* The NODE in PARENT_PATH always lives in POOL, i.e. it will - * survive the cleanup of ITERPOOL and the DAG cache.*/ - here = parent_path->node; - - /* Parse out the next entry from the path. */ - entry = svn_fs__next_entry_name(&next, rest, pool); - - /* Update the path traversed thus far. */ - path_so_far->data[path_so_far->len] = '/'; - path_so_far->len += strlen(entry) + 1; - path_so_far->data[path_so_far->len] = '\0'; - - /* Given the behavior of svn_fs__next_entry_name(), ENTRY may be an - empty string when the path either starts or ends with a slash. - In either case, we stay put: the current directory stays the - same, and we add nothing to the parent path. We only need to - process non-empty path segments. */ - if (*entry != '\0') - { - copy_id_inherit_t inherit; - const char *copy_path = NULL; - dag_node_t *cached_node = NULL; - - /* If we found a directory entry, follow it. First, we - check our node cache, and, failing that, we hit the DAG - layer. Don't bother to contact the cache for the last - element if we already know the lookup to fail for the - complete path. */ - if (next || !(flags & open_path_uncached)) - SVN_ERR(dag_node_cache_get(&cached_node, root, path_so_far->data, - pool)); - if (cached_node) - child = cached_node; - else - SVN_ERR(svn_fs_x__dag_open(&child, here, entry, pool, iterpool)); - - /* "file not found" requires special handling. */ - if (child == NULL) - { - /* If this was the last path component, and the caller - said it was optional, then don't return an error; - just put a NULL node pointer in the path. */ - - if ((flags & open_path_last_optional) - && (! next || *next == '\0')) - { - parent_path = make_parent_path(NULL, entry, parent_path, - pool); - break; - } - else if (flags & open_path_allow_null) - { - parent_path = NULL; - break; - } - else - { - /* Build a better error message than svn_fs_x__dag_open - can provide, giving the root and full path name. */ - return SVN_FS__NOT_FOUND(root, path); - } - } - - if (flags & open_path_node_only) - { - /* Shortcut: the caller only wants the final DAG node. */ - parent_path->node = svn_fs_x__dag_copy_into_pool(child, pool); - } - else - { - /* Now, make a parent_path item for CHILD. */ - parent_path = make_parent_path(child, entry, parent_path, pool); - if (is_txn_path) - { - SVN_ERR(get_copy_inheritance(&inherit, ©_path, fs, - parent_path, iterpool)); - parent_path->copy_inherit = inherit; - parent_path->copy_src_path = apr_pstrdup(pool, copy_path); - } - } - - /* Cache the node we found (if it wasn't already cached). */ - if (! cached_node) - SVN_ERR(dag_node_cache_set(root, path_so_far->data, child, - iterpool)); - } - - /* Are we finished traversing the path? */ - if (! next) - break; - - /* The path isn't finished yet; we'd better be in a directory. */ - if (svn_fs_x__dag_node_kind(child) != svn_node_dir) - SVN_ERR_W(SVN_FS__ERR_NOT_DIRECTORY(fs, path_so_far->data), - apr_psprintf(iterpool, _("Failure opening '%s'"), path)); - - rest = next; - } - - svn_pool_destroy(iterpool); - *parent_path_p = parent_path; - return SVN_NO_ERROR; -} - - -/* Make the node referred to by PARENT_PATH mutable, if it isn't already, - allocating from RESULT_POOL. ROOT must be the root from which - PARENT_PATH descends. Clone any parent directories as needed. - Adjust the dag nodes in PARENT_PATH to refer to the clones. Use - ERROR_PATH in error messages. Use SCRATCH_POOL for temporaries. */ -static svn_error_t * -make_path_mutable(svn_fs_root_t *root, - parent_path_t *parent_path, - const char *error_path, - apr_pool_t *result_pool, - apr_pool_t *scratch_pool) -{ - dag_node_t *clone; - svn_fs_x__txn_id_t txn_id = root_txn_id(root); - - /* Is the node mutable already? */ - if (svn_fs_x__dag_check_mutable(parent_path->node)) - return SVN_NO_ERROR; - - /* Are we trying to clone the root, or somebody's child node? */ - if (parent_path->parent) - { - svn_fs_x__id_t copy_id = { SVN_INVALID_REVNUM, 0 }; - svn_fs_x__id_t *copy_id_ptr = ©_id; - copy_id_inherit_t inherit = parent_path->copy_inherit; - const char *clone_path, *copyroot_path; - svn_revnum_t copyroot_rev; - svn_boolean_t is_parent_copyroot = FALSE; - svn_fs_root_t *copyroot_root; - dag_node_t *copyroot_node; - svn_boolean_t related; - - /* We're trying to clone somebody's child. Make sure our parent - is mutable. */ - SVN_ERR(make_path_mutable(root, parent_path->parent, - error_path, result_pool, scratch_pool)); - - switch (inherit) - { - case copy_id_inherit_parent: - SVN_ERR(svn_fs_x__dag_get_copy_id(©_id, - parent_path->parent->node)); - break; - - case copy_id_inherit_new: - SVN_ERR(svn_fs_x__reserve_copy_id(©_id, root->fs, txn_id, - scratch_pool)); - break; - - case copy_id_inherit_self: - copy_id_ptr = NULL; - break; - - case copy_id_inherit_unknown: - default: - SVN_ERR_MALFUNCTION(); /* uh-oh -- somebody didn't calculate copy-ID - inheritance data. */ - } - - /* Determine what copyroot our new child node should use. */ - SVN_ERR(svn_fs_x__dag_get_copyroot(©root_rev, ©root_path, - parent_path->node)); - SVN_ERR(svn_fs_x__revision_root(©root_root, root->fs, - copyroot_rev, scratch_pool)); - SVN_ERR(get_dag(©root_node, copyroot_root, copyroot_path, - result_pool)); - - SVN_ERR(svn_fs_x__dag_related_node(&related, copyroot_node, - parent_path->node)); - if (!related) - is_parent_copyroot = TRUE; - - /* Now make this node mutable. */ - clone_path = parent_path_path(parent_path->parent, scratch_pool); - SVN_ERR(svn_fs_x__dag_clone_child(&clone, - parent_path->parent->node, - clone_path, - parent_path->entry, - copy_id_ptr, txn_id, - is_parent_copyroot, - result_pool, - scratch_pool)); - - /* Update the path cache. */ - SVN_ERR(dag_node_cache_set(root, - parent_path_path(parent_path, scratch_pool), - clone, scratch_pool)); - } - else - { - /* We're trying to clone the root directory. */ - SVN_ERR(mutable_root_node(&clone, root, error_path, result_pool, - scratch_pool)); - } - - /* Update the PARENT_PATH link to refer to the clone. */ - parent_path->node = clone; - - return SVN_NO_ERROR; -} - - -/* Open the node identified by PATH in ROOT. Set DAG_NODE_P to the - node we find, allocated in POOL. Return the error - SVN_ERR_FS_NOT_FOUND if this node doesn't exist. - */ -static svn_error_t * -get_dag(dag_node_t **dag_node_p, - svn_fs_root_t *root, - const char *path, - apr_pool_t *pool) -{ - parent_path_t *parent_path; - dag_node_t *node = NULL; - - /* First we look for the DAG in our cache - (if the path may be canonical). */ - if (*path == '/') - SVN_ERR(dag_node_cache_get(&node, root, path, pool)); - - if (! node) - { - /* Canonicalize the input PATH. As it turns out, >95% of all paths - * seen here during e.g. svnadmin verify are non-canonical, i.e. - * miss the leading '/'. Unconditional canonicalization has a net - * performance benefit over previously checking path for being - * canonical. */ - path = svn_fs__canonicalize_abspath(path, pool); - SVN_ERR(dag_node_cache_get(&node, root, path, pool)); - - if (! node) - { - /* Call open_path with no flags, as we want this to return an - * error if the node for which we are searching doesn't exist. */ - SVN_ERR(open_path(&parent_path, root, path, - open_path_uncached | open_path_node_only, - FALSE, pool)); - node = parent_path->node; - - /* No need to cache our find -- open_path() will do that for us. */ - } - } - - *dag_node_p = svn_fs_x__dag_copy_into_pool(node, pool); - return SVN_NO_ERROR; -} - /* Populating the `changes' table. */ /* Add a change to the changes table in FS, keyed on transaction id TXN_ID, and indicated that a change of kind CHANGE_KIND occurred on - PATH (whose node revision id is--or was, in the case of a - deletion--NODEREV_ID), and optionally that TEXT_MODs, PROP_MODs or - MERGEINFO_MODs occurred. If the change resulted from a copy, - COPYFROM_REV and COPYFROM_PATH specify under which revision and path - the node was copied from. If this was not part of a copy, COPYFROM_REV - should be SVN_INVALID_REVNUM. Use SCRATCH_POOL for temporary allocations. + PATH, and optionally that TEXT_MODs, PROP_MODs or MERGEINFO_MODs + occurred. If the change resulted from a copy, COPYFROM_REV and + COPYFROM_PATH specify under which revision and path the node was + copied from. If this was not part of a copy, COPYFROM_REV should + be SVN_INVALID_REVNUM. Use SCRATCH_POOL for temporary allocations. */ static svn_error_t * add_change(svn_fs_t *fs, svn_fs_x__txn_id_t txn_id, const char *path, - const svn_fs_x__id_t *noderev_id, svn_fs_path_change_kind_t change_kind, svn_boolean_t text_mod, svn_boolean_t prop_mod, @@ -1275,8 +246,7 @@ add_change(svn_fs_t *fs, return svn_fs_x__add_change(fs, txn_id, svn_fs__canonicalize_abspath(path, scratch_pool), - noderev_id, change_kind, - text_mod, prop_mod, mergeinfo_mod, + change_kind, text_mod, prop_mod, mergeinfo_mod, node_kind, copyfrom_rev, copyfrom_path, scratch_pool); } @@ -1306,10 +276,12 @@ x_node_id(const svn_fs_id_t **id_p, } else { + apr_pool_t *scratch_pool = svn_pool_create(pool); dag_node_t *node; - SVN_ERR(get_dag(&node, root, path, pool)); + SVN_ERR(svn_fs_x__get_temp_dag_node(&node, root, path, scratch_pool)); noderev_id = *svn_fs_x__dag_get_id(node); + svn_pool_destroy(scratch_pool); } *id_p = svn_fs_x__id_create(svn_fs_x__id_create_context(root->fs, pool), @@ -1361,13 +333,13 @@ x_node_relation(svn_fs_node_relation_t *relation, /* We checked for all separations between ID spaces (repos, txn). * Now, we can simply test for the ID values themselves. */ - SVN_ERR(get_dag(&node, root_a, path_a, scratch_pool)); + SVN_ERR(svn_fs_x__get_temp_dag_node(&node, root_a, path_a, scratch_pool)); noderev_id_a = *svn_fs_x__dag_get_id(node); - SVN_ERR(svn_fs_x__dag_get_node_id(&node_id_a, node)); + node_id_a = *svn_fs_x__dag_get_node_id(node); - SVN_ERR(get_dag(&node, root_b, path_b, scratch_pool)); + SVN_ERR(svn_fs_x__get_temp_dag_node(&node, root_b, path_b, scratch_pool)); noderev_id_b = *svn_fs_x__dag_get_id(node); - SVN_ERR(svn_fs_x__dag_get_node_id(&node_id_b, node)); + node_id_b = *svn_fs_x__dag_get_node_id(node); /* In FSX, even in-txn IDs are globally unique. * So, we can simply compare them. */ @@ -1389,7 +361,7 @@ svn_fs_x__node_created_rev(svn_revnum_t *revision, { dag_node_t *node; - SVN_ERR(get_dag(&node, root, path, scratch_pool)); + SVN_ERR(svn_fs_x__get_temp_dag_node(&node, root, path, scratch_pool)); *revision = svn_fs_x__dag_get_revision(node); return SVN_NO_ERROR; @@ -1406,28 +378,8 @@ x_node_created_path(const char **created_path, { dag_node_t *node; - SVN_ERR(get_dag(&node, root, path, pool)); - *created_path = svn_fs_x__dag_get_created_path(node); - - return SVN_NO_ERROR; -} - - -/* Set *KIND_P to the type of node located at PATH under ROOT. - Perform temporary allocations in SCRATCH_POOL. */ -static svn_error_t * -node_kind(svn_node_kind_t *kind_p, - svn_fs_root_t *root, - const char *path, - apr_pool_t *scratch_pool) -{ - dag_node_t *node; - - /* Get the node id. */ - SVN_ERR(get_dag(&node, root, path, scratch_pool)); - - /* Use the node id to get the real kind. */ - *kind_p = svn_fs_x__dag_node_kind(node); + SVN_ERR(svn_fs_x__get_temp_dag_node(&node, root, path, pool)); + *created_path = apr_pstrdup(pool, svn_fs_x__dag_get_created_path(node)); return SVN_NO_ERROR; } @@ -1442,7 +394,16 @@ svn_fs_x__check_path(svn_node_kind_t *kind_p, const char *path, apr_pool_t *scratch_pool) { - svn_error_t *err = node_kind(kind_p, root, path, scratch_pool); + dag_node_t *node; + + /* Get the node id. */ + svn_error_t *err = svn_fs_x__get_temp_dag_node(&node, root, path, + scratch_pool); + + /* Use the node id to get the real kind. */ + if (!err) + *kind_p = svn_fs_x__dag_node_kind(node); + if (err && ((err->apr_err == SVN_ERR_FS_NOT_FOUND) || (err->apr_err == SVN_ERR_FS_NOT_DIRECTORY))) @@ -1469,11 +430,12 @@ x_node_prop(svn_string_t **value_p, apr_hash_t *proplist; apr_pool_t *scratch_pool = svn_pool_create(pool); - SVN_ERR(get_dag(&node, root, path, pool)); - SVN_ERR(svn_fs_x__dag_get_proplist(&proplist, node, pool, scratch_pool)); + SVN_ERR(svn_fs_x__get_temp_dag_node(&node, root, path, scratch_pool)); + SVN_ERR(svn_fs_x__dag_get_proplist(&proplist, node, scratch_pool, + scratch_pool)); *value_p = NULL; if (proplist) - *value_p = svn_hash_gets(proplist, propname); + *value_p = svn_string_dup(svn_hash_gets(proplist, propname), pool); svn_pool_destroy(scratch_pool); return SVN_NO_ERROR; @@ -1493,7 +455,7 @@ x_node_proplist(apr_hash_t **table_p, dag_node_t *node; apr_pool_t *scratch_pool = svn_pool_create(pool); - SVN_ERR(get_dag(&node, root, path, pool)); + SVN_ERR(svn_fs_x__get_temp_dag_node(&node, root, path, scratch_pool)); SVN_ERR(svn_fs_x__dag_get_proplist(table_p, node, pool, scratch_pool)); svn_pool_destroy(scratch_pool); @@ -1516,7 +478,7 @@ x_node_has_props(svn_boolean_t *has_props, } static svn_error_t * -increment_mergeinfo_up_tree(parent_path_t *pp, +increment_mergeinfo_up_tree(svn_fs_x__dag_path_t *pp, apr_int64_t increment, apr_pool_t *scratch_pool) { @@ -1546,7 +508,7 @@ x_change_node_prop(svn_fs_root_t *root, const svn_string_t *value, apr_pool_t *scratch_pool) { - parent_path_t *parent_path; + svn_fs_x__dag_path_t *dag_path; apr_hash_t *proplist; svn_fs_x__txn_id_t txn_id; svn_boolean_t mergeinfo_mod = FALSE; @@ -1554,10 +516,10 @@ x_change_node_prop(svn_fs_root_t *root, if (! root->is_txn_root) return SVN_FS__NOT_TXN(root); - txn_id = root_txn_id(root); + txn_id = svn_fs_x__root_txn_id(root); - path = svn_fs__canonicalize_abspath(path, subpool); - SVN_ERR(open_path(&parent_path, root, path, 0, TRUE, subpool)); + SVN_ERR(svn_fs_x__get_dag_path(&dag_path, root, path, 0, TRUE, subpool, + subpool)); /* Check (non-recursively) to see if path is locked; if so, check that we can use it. */ @@ -1565,8 +527,9 @@ x_change_node_prop(svn_fs_root_t *root, SVN_ERR(svn_fs_x__allow_locked_operation(path, root->fs, FALSE, FALSE, subpool)); - SVN_ERR(make_path_mutable(root, parent_path, path, subpool, subpool)); - SVN_ERR(svn_fs_x__dag_get_proplist(&proplist, parent_path->node, subpool, + SVN_ERR(svn_fs_x__make_path_mutable(root, dag_path, path, subpool, + subpool)); + SVN_ERR(svn_fs_x__dag_get_proplist(&proplist, dag_path->node, subpool, subpool)); /* If there's no proplist, but we're just deleting a property, exit now. */ @@ -1580,8 +543,8 @@ x_change_node_prop(svn_fs_root_t *root, if (strcmp(name, SVN_PROP_MERGEINFO) == 0) { apr_int64_t increment = 0; - svn_boolean_t had_mergeinfo; - SVN_ERR(svn_fs_x__dag_has_mergeinfo(&had_mergeinfo, parent_path->node)); + svn_boolean_t had_mergeinfo + = svn_fs_x__dag_has_mergeinfo(dag_path->node); if (value && !had_mergeinfo) increment = 1; @@ -1590,8 +553,8 @@ x_change_node_prop(svn_fs_root_t *root, if (increment != 0) { - SVN_ERR(increment_mergeinfo_up_tree(parent_path, increment, subpool)); - SVN_ERR(svn_fs_x__dag_set_has_mergeinfo(parent_path->node, + SVN_ERR(increment_mergeinfo_up_tree(dag_path, increment, subpool)); + SVN_ERR(svn_fs_x__dag_set_has_mergeinfo(dag_path->node, (value != NULL), subpool)); } @@ -1602,14 +565,13 @@ x_change_node_prop(svn_fs_root_t *root, svn_hash_sets(proplist, name, value); /* Overwrite the node's proplist. */ - SVN_ERR(svn_fs_x__dag_set_proplist(parent_path->node, proplist, + SVN_ERR(svn_fs_x__dag_set_proplist(dag_path->node, proplist, subpool)); /* Make a record of this modification in the changes table. */ SVN_ERR(add_change(root->fs, txn_id, path, - svn_fs_x__dag_get_id(parent_path->node), svn_fs_path_change_modify, FALSE, TRUE, mergeinfo_mod, - svn_fs_x__dag_node_kind(parent_path->node), + svn_fs_x__dag_node_kind(dag_path->node), SVN_INVALID_REVNUM, NULL, subpool)); svn_pool_destroy(subpool); @@ -1639,8 +601,8 @@ x_props_changed(svn_boolean_t *changed_p, (SVN_ERR_FS_GENERAL, NULL, _("Cannot compare property value between two different filesystems")); - SVN_ERR(get_dag(&node1, root1, path1, subpool)); - SVN_ERR(get_dag(&node2, root2, path2, subpool)); + SVN_ERR(svn_fs_x__get_dag_node(&node1, root1, path1, subpool, subpool)); + SVN_ERR(svn_fs_x__get_temp_dag_node(&node2, root2, path2, subpool)); SVN_ERR(svn_fs_x__dag_things_different(changed_p, NULL, node1, node2, strict, subpool)); svn_pool_destroy(subpool); @@ -1654,9 +616,12 @@ x_props_changed(svn_boolean_t *changed_p, /* Set *NODE to the root node of ROOT. */ static svn_error_t * -get_root(dag_node_t **node, svn_fs_root_t *root, apr_pool_t *pool) +get_root(dag_node_t **node, + svn_fs_root_t *root, + apr_pool_t *result_pool, + apr_pool_t *scratch_pool) { - return get_dag(node, root, "/", pool); + return svn_fs_x__get_dag_node(node, root, "/", result_pool, scratch_pool); } @@ -1714,7 +679,6 @@ compare_dir_structure(svn_boolean_t *changed, if (strcmp(lhs_entry->name, rhs_entry->name) == 0) { - svn_boolean_t same_history; dag_node_t *lhs_node, *rhs_node; /* Unchanged entry? */ @@ -1729,9 +693,7 @@ compare_dir_structure(svn_boolean_t *changed, iterpool, iterpool)); SVN_ERR(svn_fs_x__dag_get_node(&rhs_node, fs, &rhs_entry->id, iterpool, iterpool)); - SVN_ERR(svn_fs_x__dag_same_line_of_history(&same_history, - lhs_node, rhs_node)); - if (same_history) + if (svn_fs_x__dag_same_line_of_history(lhs_node, rhs_node)) continue; } @@ -1974,15 +936,11 @@ merge(svn_stringbuf_t *conflict_p, process, but the transaction did not touch this entry. */ else if (t_entry && svn_fs_x__id_eq(&a_entry->id, &t_entry->id)) { - apr_int64_t mergeinfo_start; - apr_int64_t mergeinfo_end; - dag_node_t *t_ent_node; SVN_ERR(svn_fs_x__dag_get_node(&t_ent_node, fs, &t_entry->id, iterpool, iterpool)); - SVN_ERR(svn_fs_x__dag_get_mergeinfo_count(&mergeinfo_start, - t_ent_node)); - mergeinfo_increment -= mergeinfo_start; + mergeinfo_increment + -= svn_fs_x__dag_get_mergeinfo_count(t_ent_node); if (s_entry) { @@ -1990,9 +948,8 @@ merge(svn_stringbuf_t *conflict_p, SVN_ERR(svn_fs_x__dag_get_node(&s_ent_node, fs, &s_entry->id, iterpool, iterpool)); - SVN_ERR(svn_fs_x__dag_get_mergeinfo_count(&mergeinfo_end, - s_ent_node)); - mergeinfo_increment += mergeinfo_end; + mergeinfo_increment + += svn_fs_x__dag_get_mergeinfo_count(s_ent_node); SVN_ERR(svn_fs_x__dag_set_entry(target, a_entry->name, &s_entry->id, @@ -2015,7 +972,6 @@ merge(svn_stringbuf_t *conflict_p, dag_node_t *s_ent_node, *t_ent_node, *a_ent_node; const char *new_tpath; apr_int64_t sub_mergeinfo_increment; - svn_boolean_t s_a_same, t_a_same; /* If SOURCE-ENTRY and TARGET-ENTRY are both null, that's a double delete; if one of them is null, that's a delete versus @@ -2045,11 +1001,8 @@ merge(svn_stringbuf_t *conflict_p, /* If either SOURCE-ENTRY or TARGET-ENTRY is not a direct modification of ANCESTOR-ENTRY, declare a conflict. */ - SVN_ERR(svn_fs_x__dag_same_line_of_history(&s_a_same, s_ent_node, - a_ent_node)); - SVN_ERR(svn_fs_x__dag_same_line_of_history(&t_a_same, t_ent_node, - a_ent_node)); - if (!s_a_same || !t_a_same) + if ( !svn_fs_x__dag_same_line_of_history(s_ent_node, a_ent_node) + || !svn_fs_x__dag_same_line_of_history(t_ent_node, a_ent_node)) return conflict_err(conflict_p, svn_fspath__join(target_path, a_entry->name, @@ -2073,7 +1026,6 @@ merge(svn_stringbuf_t *conflict_p, { svn_fs_x__dirent_t *a_entry, *s_entry, *t_entry; dag_node_t *s_ent_node; - apr_int64_t mergeinfo_s; svn_pool_clear(iterpool); @@ -2094,8 +1046,7 @@ merge(svn_stringbuf_t *conflict_p, SVN_ERR(svn_fs_x__dag_get_node(&s_ent_node, fs, &s_entry->id, iterpool, iterpool)); - SVN_ERR(svn_fs_x__dag_get_mergeinfo_count(&mergeinfo_s, s_ent_node)); - mergeinfo_increment += mergeinfo_s; + mergeinfo_increment += svn_fs_x__dag_get_mergeinfo_count(s_ent_node); SVN_ERR(svn_fs_x__dag_set_entry (target, s_entry->name, &s_entry->id, s_entry->kind, @@ -2135,21 +1086,21 @@ merge_changes(dag_node_t *ancestor_node, dag_node_t *txn_root_node; svn_fs_t *fs = txn->fs; svn_fs_x__txn_id_t txn_id = svn_fs_x__txn_get_id(txn); - svn_boolean_t related; - SVN_ERR(svn_fs_x__dag_txn_root(&txn_root_node, fs, txn_id, scratch_pool, - scratch_pool)); + SVN_ERR(svn_fs_x__dag_root(&txn_root_node, fs, + svn_fs_x__change_set_by_txn(txn_id), + scratch_pool, scratch_pool)); if (ancestor_node == NULL) { svn_revnum_t base_rev; SVN_ERR(svn_fs_x__get_base_rev(&base_rev, fs, txn_id, scratch_pool)); - SVN_ERR(svn_fs_x__dag_revision_root(&ancestor_node, fs, base_rev, - scratch_pool, scratch_pool)); + SVN_ERR(svn_fs_x__dag_root(&ancestor_node, fs, + svn_fs_x__change_set_by_rev(base_rev), + scratch_pool, scratch_pool)); } - SVN_ERR(svn_fs_x__dag_related_node(&related, ancestor_node, txn_root_node)); - if (!related) + if (!svn_fs_x__dag_related_node(ancestor_node, txn_root_node)) { /* If no changes have been made in TXN since its current base, then it can't conflict with any changes since that base. @@ -2249,7 +1200,8 @@ svn_fs_x__commit_txn(const char **conflict_p, note that the youngest rev may have changed by then -- that's why we're careful to get this root in its own bdb txn here). */ - SVN_ERR(get_root(&youngish_root_node, youngish_root, iterpool)); + SVN_ERR(get_root(&youngish_root_node, youngish_root, iterpool, + iterpool)); /* Try to merge. If the merge succeeds, the base root node of TARGET's txn will become the same as youngish_root_node, so @@ -2299,7 +1251,7 @@ svn_fs_x__commit_txn(const char **conflict_p, if (ffd->pack_after_commit) { - SVN_ERR(svn_fs_x__pack(fs, NULL, NULL, NULL, NULL, pool)); + SVN_ERR(svn_fs_x__pack(fs, 0, NULL, NULL, NULL, NULL, pool)); } return SVN_NO_ERROR; @@ -2350,10 +1302,10 @@ x_merge(const char **conflict_p, */ /* Get the ancestor node. */ - SVN_ERR(get_root(&ancestor, ancestor_root, pool)); + SVN_ERR(get_root(&ancestor, ancestor_root, pool, pool)); /* Get the source node. */ - SVN_ERR(get_root(&source, source_root, pool)); + SVN_ERR(get_root(&source, source_root, pool, pool)); /* Open a txn for the txn root into which we're merging. */ SVN_ERR(svn_fs_x__open_txn(&txn, ancestor_root->fs, target_root->txn, @@ -2404,7 +1356,7 @@ x_dir_entries(apr_hash_t **table_p, apr_pool_t *scratch_pool = svn_pool_create(pool); /* Get the entries for this path in the caller's pool. */ - SVN_ERR(get_dag(&node, root, path, scratch_pool)); + SVN_ERR(svn_fs_x__get_temp_dag_node(&node, root, path, scratch_pool)); SVN_ERR(svn_fs_x__dag_dir_entries(&table, node, scratch_pool, scratch_pool)); @@ -2454,14 +1406,14 @@ x_make_dir(svn_fs_root_t *root, const char *path, apr_pool_t *scratch_pool) { - parent_path_t *parent_path; + svn_fs_x__dag_path_t *dag_path; dag_node_t *sub_dir; - svn_fs_x__txn_id_t txn_id = root_txn_id(root); + svn_fs_x__txn_id_t txn_id = svn_fs_x__root_txn_id(root); apr_pool_t *subpool = svn_pool_create(scratch_pool); - path = svn_fs__canonicalize_abspath(path, subpool); - SVN_ERR(open_path(&parent_path, root, path, open_path_last_optional, - TRUE, subpool)); + SVN_ERR(svn_fs_x__get_dag_path(&dag_path, root, path, + svn_fs_x__dag_path_last_optional, + TRUE, subpool, subpool)); /* Check (recursively) to see if some lock is 'reserving' a path at that location, or even some child-path; if so, check that we can @@ -2472,26 +1424,25 @@ x_make_dir(svn_fs_root_t *root, /* If there's already a sub-directory by that name, complain. This also catches the case of trying to make a subdirectory named `/'. */ - if (parent_path->node) + if (dag_path->node) return SVN_FS__ALREADY_EXISTS(root, path); /* Create the subdirectory. */ - SVN_ERR(make_path_mutable(root, parent_path->parent, path, subpool, - subpool)); + SVN_ERR(svn_fs_x__make_path_mutable(root, dag_path->parent, path, subpool, + subpool)); SVN_ERR(svn_fs_x__dag_make_dir(&sub_dir, - parent_path->parent->node, - parent_path_path(parent_path->parent, + dag_path->parent->node, + parent_path_path(dag_path->parent, subpool), - parent_path->entry, + dag_path->entry, txn_id, subpool, subpool)); /* Add this directory to the path cache. */ - SVN_ERR(dag_node_cache_set(root, parent_path_path(parent_path, subpool), - sub_dir, subpool)); + svn_fs_x__update_dag_cache(sub_dir); /* Make a record of this modification in the changes table. */ - SVN_ERR(add_change(root->fs, txn_id, path, svn_fs_x__dag_get_id(sub_dir), + SVN_ERR(add_change(root->fs, txn_id, path, svn_fs_path_change_add, FALSE, FALSE, FALSE, svn_node_dir, SVN_INVALID_REVNUM, NULL, subpool)); @@ -2507,7 +1458,7 @@ x_delete_node(svn_fs_root_t *root, const char *path, apr_pool_t *scratch_pool) { - parent_path_t *parent_path; + svn_fs_x__dag_path_t *dag_path; svn_fs_x__txn_id_t txn_id; apr_int64_t mergeinfo_count = 0; svn_node_kind_t kind; @@ -2516,13 +1467,13 @@ x_delete_node(svn_fs_root_t *root, if (! root->is_txn_root) return SVN_FS__NOT_TXN(root); - txn_id = root_txn_id(root); - path = svn_fs__canonicalize_abspath(path, subpool); - SVN_ERR(open_path(&parent_path, root, path, 0, TRUE, subpool)); - kind = svn_fs_x__dag_node_kind(parent_path->node); + txn_id = svn_fs_x__root_txn_id(root); + SVN_ERR(svn_fs_x__get_dag_path(&dag_path, root, path, 0, TRUE, subpool, + subpool)); + kind = svn_fs_x__dag_node_kind(dag_path->node); /* We can't remove the root of the filesystem. */ - if (! parent_path->parent) + if (! dag_path->parent) return svn_error_create(SVN_ERR_FS_ROOT_DIR, NULL, _("The root directory cannot be deleted")); @@ -2533,28 +1484,24 @@ x_delete_node(svn_fs_root_t *root, subpool)); /* Make the parent directory mutable, and do the deletion. */ - SVN_ERR(make_path_mutable(root, parent_path->parent, path, subpool, - subpool)); - SVN_ERR(svn_fs_x__dag_get_mergeinfo_count(&mergeinfo_count, - parent_path->node)); - SVN_ERR(svn_fs_x__dag_delete(parent_path->parent->node, - parent_path->entry, + SVN_ERR(svn_fs_x__make_path_mutable(root, dag_path->parent, path, subpool, + subpool)); + mergeinfo_count = svn_fs_x__dag_get_mergeinfo_count(dag_path->node); + SVN_ERR(svn_fs_x__dag_delete(dag_path->parent->node, + dag_path->entry, txn_id, subpool)); /* Remove this node and any children from the path cache. */ - SVN_ERR(dag_node_cache_invalidate(root, parent_path_path(parent_path, - subpool), - subpool)); + svn_fs_x__invalidate_dag_cache(root, parent_path_path(dag_path, subpool)); /* Update mergeinfo counts for parents */ if (mergeinfo_count > 0) - SVN_ERR(increment_mergeinfo_up_tree(parent_path->parent, + SVN_ERR(increment_mergeinfo_up_tree(dag_path->parent, -mergeinfo_count, subpool)); /* Make a record of this modification in the changes table. */ SVN_ERR(add_change(root->fs, txn_id, path, - svn_fs_x__dag_get_id(parent_path->node), svn_fs_path_change_delete, FALSE, FALSE, FALSE, kind, SVN_INVALID_REVNUM, NULL, subpool)); @@ -2588,8 +1535,8 @@ copy_helper(svn_fs_root_t *from_root, apr_pool_t *scratch_pool) { dag_node_t *from_node; - parent_path_t *to_parent_path; - svn_fs_x__txn_id_t txn_id = root_txn_id(to_root); + svn_fs_x__dag_path_t *to_dag_path; + svn_fs_x__txn_id_t txn_id = svn_fs_x__root_txn_id(to_root); svn_boolean_t same_p; /* Use an error check, not an assert, because even the caller cannot @@ -2613,13 +1560,15 @@ copy_helper(svn_fs_root_t *from_root, _("Copy immutable tree not supported")); /* Get the NODE for FROM_PATH in FROM_ROOT.*/ - SVN_ERR(get_dag(&from_node, from_root, from_path, scratch_pool)); + SVN_ERR(svn_fs_x__get_dag_node(&from_node, from_root, from_path, + scratch_pool, scratch_pool)); /* Build up the parent path from TO_PATH in TO_ROOT. If the last component does not exist, it's not that big a deal. We'll just make one there. */ - SVN_ERR(open_path(&to_parent_path, to_root, to_path, - open_path_last_optional, TRUE, scratch_pool)); + SVN_ERR(svn_fs_x__get_dag_path(&to_dag_path, to_root, to_path, + svn_fs_x__dag_path_last_optional, TRUE, + scratch_pool, scratch_pool)); /* Check to see if path (or any child thereof) is locked; if so, check that we can use the existing lock(s). */ @@ -2631,9 +1580,9 @@ copy_helper(svn_fs_root_t *from_root, source (in other words, this operation would result in nothing happening at all), just do nothing an return successfully, proud that you saved yourself from a tiresome task. */ - if (to_parent_path->node && + if (to_dag_path->node && svn_fs_x__id_eq(svn_fs_x__dag_get_id(from_node), - svn_fs_x__dag_get_id(to_parent_path->node))) + svn_fs_x__dag_get_id(to_dag_path->node))) return SVN_NO_ERROR; if (! from_root->is_txn_root) @@ -2646,11 +1595,11 @@ copy_helper(svn_fs_root_t *from_root, /* If TO_PATH already existed prior to the copy, note that this operation is a replacement, not an addition. */ - if (to_parent_path->node) + if (to_dag_path->node) { kind = svn_fs_path_change_replace; - SVN_ERR(svn_fs_x__dag_get_mergeinfo_count(&mergeinfo_start, - to_parent_path->node)); + mergeinfo_start + = svn_fs_x__dag_get_mergeinfo_count(to_dag_path->node); } else { @@ -2658,17 +1607,18 @@ copy_helper(svn_fs_root_t *from_root, mergeinfo_start = 0; } - SVN_ERR(svn_fs_x__dag_get_mergeinfo_count(&mergeinfo_end, from_node)); + mergeinfo_end = svn_fs_x__dag_get_mergeinfo_count(from_node); /* Make sure the target node's parents are mutable. */ - SVN_ERR(make_path_mutable(to_root, to_parent_path->parent, - to_path, scratch_pool, scratch_pool)); + SVN_ERR(svn_fs_x__make_path_mutable(to_root, to_dag_path->parent, + to_path, scratch_pool, + scratch_pool)); /* Canonicalize the copyfrom path. */ from_canonpath = svn_fs__canonicalize_abspath(from_path, scratch_pool); - SVN_ERR(svn_fs_x__dag_copy(to_parent_path->parent->node, - to_parent_path->entry, + SVN_ERR(svn_fs_x__dag_copy(to_dag_path->parent->node, + to_dag_path->entry, from_node, preserve_history, from_root->rev, @@ -2676,20 +1626,19 @@ copy_helper(svn_fs_root_t *from_root, txn_id, scratch_pool)); if (kind != svn_fs_path_change_add) - SVN_ERR(dag_node_cache_invalidate(to_root, - parent_path_path(to_parent_path, - scratch_pool), - scratch_pool)); + svn_fs_x__invalidate_dag_cache(to_root, + parent_path_path(to_dag_path, + scratch_pool)); if (mergeinfo_start != mergeinfo_end) - SVN_ERR(increment_mergeinfo_up_tree(to_parent_path->parent, + SVN_ERR(increment_mergeinfo_up_tree(to_dag_path->parent, mergeinfo_end - mergeinfo_start, scratch_pool)); /* Make a record of this modification in the changes table. */ - SVN_ERR(get_dag(&new_node, to_root, to_path, scratch_pool)); - SVN_ERR(add_change(to_root->fs, txn_id, to_path, - svn_fs_x__dag_get_id(new_node), kind, FALSE, + SVN_ERR(svn_fs_x__get_dag_node(&new_node, to_root, to_path, + scratch_pool, scratch_pool)); + SVN_ERR(add_change(to_root->fs, txn_id, to_path, kind, FALSE, FALSE, FALSE, svn_fs_x__dag_node_kind(from_node), from_root->rev, from_canonpath, scratch_pool)); } @@ -2775,11 +1724,10 @@ x_copied_from(svn_revnum_t *rev_p, { dag_node_t *node; - /* There is no cached entry, look it up the old-fashioned - way. */ - SVN_ERR(get_dag(&node, root, path, pool)); - SVN_ERR(svn_fs_x__dag_get_copyfrom_rev(rev_p, node)); - SVN_ERR(svn_fs_x__dag_get_copyfrom_path(path_p, node)); + /* There is no cached entry, look it up the old-fashioned way. */ + SVN_ERR(svn_fs_x__get_temp_dag_node(&node, root, path, pool)); + *rev_p = svn_fs_x__dag_get_copyfrom_rev(node); + *path_p = svn_fs_x__dag_get_copyfrom_path(node); return SVN_NO_ERROR; } @@ -2795,18 +1743,18 @@ x_make_file(svn_fs_root_t *root, const char *path, apr_pool_t *scratch_pool) { - parent_path_t *parent_path; + svn_fs_x__dag_path_t *dag_path; dag_node_t *child; - svn_fs_x__txn_id_t txn_id = root_txn_id(root); + svn_fs_x__txn_id_t txn_id = svn_fs_x__root_txn_id(root); apr_pool_t *subpool = svn_pool_create(scratch_pool); - path = svn_fs__canonicalize_abspath(path, subpool); - SVN_ERR(open_path(&parent_path, root, path, open_path_last_optional, - TRUE, subpool)); + SVN_ERR(svn_fs_x__get_dag_path(&dag_path, root, path, + svn_fs_x__dag_path_last_optional, + TRUE, subpool, subpool)); /* If there's already a file by that name, complain. This also catches the case of trying to make a file named `/'. */ - if (parent_path->node) + if (dag_path->node) return SVN_FS__ALREADY_EXISTS(root, path); /* Check (non-recursively) to see if path is locked; if so, check @@ -2816,22 +1764,21 @@ x_make_file(svn_fs_root_t *root, subpool)); /* Create the file. */ - SVN_ERR(make_path_mutable(root, parent_path->parent, path, subpool, - subpool)); + SVN_ERR(svn_fs_x__make_path_mutable(root, dag_path->parent, path, subpool, + subpool)); SVN_ERR(svn_fs_x__dag_make_file(&child, - parent_path->parent->node, - parent_path_path(parent_path->parent, + dag_path->parent->node, + parent_path_path(dag_path->parent, subpool), - parent_path->entry, + dag_path->entry, txn_id, subpool, subpool)); /* Add this file to the path cache. */ - SVN_ERR(dag_node_cache_set(root, parent_path_path(parent_path, subpool), - child, subpool)); + svn_fs_x__update_dag_cache(child); /* Make a record of this modification in the changes table. */ - SVN_ERR(add_change(root->fs, txn_id, path, svn_fs_x__dag_get_id(child), + SVN_ERR(add_change(root->fs, txn_id, path, svn_fs_path_change_add, TRUE, FALSE, FALSE, svn_node_file, SVN_INVALID_REVNUM, NULL, subpool)); @@ -2851,7 +1798,7 @@ x_file_length(svn_filesize_t *length_p, dag_node_t *file; /* First create a dag_node_t from the root/path pair. */ - SVN_ERR(get_dag(&file, root, path, scratch_pool)); + SVN_ERR(svn_fs_x__get_temp_dag_node(&file, root, path, scratch_pool)); /* Now fetch its length */ return svn_fs_x__dag_file_length(length_p, file); @@ -2870,7 +1817,7 @@ x_file_checksum(svn_checksum_t **checksum, { dag_node_t *file; - SVN_ERR(get_dag(&file, root, path, pool)); + SVN_ERR(svn_fs_x__get_temp_dag_node(&file, root, path, pool)); return svn_fs_x__dag_file_checksum(checksum, file, kind, pool); } @@ -2889,7 +1836,7 @@ x_file_contents(svn_stream_t **contents, svn_stream_t *file_stream; /* First create a dag_node_t from the root/path pair. */ - SVN_ERR(get_dag(&node, root, path, pool)); + SVN_ERR(svn_fs_x__get_temp_dag_node(&node, root, path, pool)); /* Then create a readable stream from the dag_node_t. */ SVN_ERR(svn_fs_x__dag_get_contents(&file_stream, node, pool)); @@ -2912,7 +1859,7 @@ x_try_process_file_contents(svn_boolean_t *success, apr_pool_t *pool) { dag_node_t *node; - SVN_ERR(get_dag(&node, root, path, pool)); + SVN_ERR(svn_fs_x__get_temp_dag_node(&node, root, path, pool)); return svn_fs_x__dag_try_process_file_contents(success, node, processor, baton, pool); @@ -2983,12 +1930,13 @@ apply_textdelta(void *baton, apr_pool_t *scratch_pool) { txdelta_baton_t *tb = (txdelta_baton_t *) baton; - parent_path_t *parent_path; - svn_fs_x__txn_id_t txn_id = root_txn_id(tb->root); + svn_fs_x__dag_path_t *dag_path; + svn_fs_x__txn_id_t txn_id = svn_fs_x__root_txn_id(tb->root); /* Call open_path with no flags, as we want this to return an error if the node for which we are searching doesn't exist. */ - SVN_ERR(open_path(&parent_path, tb->root, tb->path, 0, TRUE, scratch_pool)); + SVN_ERR(svn_fs_x__get_dag_path(&dag_path, tb->root, tb->path, 0, TRUE, + scratch_pool, scratch_pool)); /* Check (non-recursively) to see if path is locked; if so, check that we can use it. */ @@ -2997,9 +1945,9 @@ apply_textdelta(void *baton, FALSE, FALSE, scratch_pool)); /* Now, make sure this path is mutable. */ - SVN_ERR(make_path_mutable(tb->root, parent_path, tb->path, scratch_pool, - scratch_pool)); - tb->node = svn_fs_x__dag_dup(parent_path->node, tb->pool); + SVN_ERR(svn_fs_x__make_path_mutable(tb->root, dag_path, tb->path, + scratch_pool, scratch_pool)); + tb->node = svn_fs_x__dag_dup(dag_path->node, tb->pool); if (tb->base_checksum) { @@ -3038,7 +1986,6 @@ apply_textdelta(void *baton, /* Make a record of this modification in the changes table. */ return add_change(tb->root->fs, txn_id, tb->path, - svn_fs_x__dag_get_id(tb->node), svn_fs_path_change_modify, TRUE, FALSE, FALSE, svn_node_file, SVN_INVALID_REVNUM, NULL, scratch_pool); } @@ -3149,12 +2096,13 @@ apply_text(void *baton, apr_pool_t *scratch_pool) { text_baton_t *tb = baton; - parent_path_t *parent_path; - svn_fs_x__txn_id_t txn_id = root_txn_id(tb->root); + svn_fs_x__dag_path_t *dag_path; + svn_fs_x__txn_id_t txn_id = svn_fs_x__root_txn_id(tb->root); /* Call open_path with no flags, as we want this to return an error if the node for which we are searching doesn't exist. */ - SVN_ERR(open_path(&parent_path, tb->root, tb->path, 0, TRUE, scratch_pool)); + SVN_ERR(svn_fs_x__get_dag_path(&dag_path, tb->root, tb->path, 0, TRUE, + scratch_pool, scratch_pool)); /* Check (non-recursively) to see if path is locked; if so, check that we can use it. */ @@ -3163,9 +2111,9 @@ apply_text(void *baton, FALSE, FALSE, scratch_pool)); /* Now, make sure this path is mutable. */ - SVN_ERR(make_path_mutable(tb->root, parent_path, tb->path, scratch_pool, - scratch_pool)); - tb->node = svn_fs_x__dag_dup(parent_path->node, tb->pool); + SVN_ERR(svn_fs_x__make_path_mutable(tb->root, dag_path, tb->path, + scratch_pool, scratch_pool)); + tb->node = svn_fs_x__dag_dup(dag_path->node, tb->pool); /* Make a writable stream for replacing the file's text. */ SVN_ERR(svn_fs_x__dag_get_edit_stream(&(tb->file_stream), tb->node, @@ -3178,7 +2126,6 @@ apply_text(void *baton, /* Make a record of this modification in the changes table. */ return add_change(tb->root->fs, txn_id, tb->path, - svn_fs_x__dag_get_id(tb->node), svn_fs_path_change_modify, TRUE, FALSE, FALSE, svn_node_file, SVN_INVALID_REVNUM, NULL, scratch_pool); } @@ -3234,23 +2181,18 @@ x_contents_changed(svn_boolean_t *changed_p, (SVN_ERR_FS_GENERAL, NULL, _("Cannot compare file contents between two different filesystems")); - /* Check that both paths are files. */ - { - svn_node_kind_t kind; - - SVN_ERR(svn_fs_x__check_path(&kind, root1, path1, subpool)); - if (kind != svn_node_file) - return svn_error_createf - (SVN_ERR_FS_GENERAL, NULL, _("'%s' is not a file"), path1); + SVN_ERR(svn_fs_x__get_dag_node(&node1, root1, path1, subpool, subpool)); + /* Make sure that path is file. */ + if (svn_fs_x__dag_node_kind(node1) != svn_node_file) + return svn_error_createf + (SVN_ERR_FS_GENERAL, NULL, _("'%s' is not a file"), path1); - SVN_ERR(svn_fs_x__check_path(&kind, root2, path2, subpool)); - if (kind != svn_node_file) - return svn_error_createf - (SVN_ERR_FS_GENERAL, NULL, _("'%s' is not a file"), path2); - } + SVN_ERR(svn_fs_x__get_temp_dag_node(&node2, root2, path2, subpool)); + /* Make sure that path is file. */ + if (svn_fs_x__dag_node_kind(node2) != svn_node_file) + return svn_error_createf + (SVN_ERR_FS_GENERAL, NULL, _("'%s' is not a file"), path2); - SVN_ERR(get_dag(&node1, root1, path1, subpool)); - SVN_ERR(get_dag(&node2, root2, path2, subpool)); SVN_ERR(svn_fs_x__dag_things_different(NULL, changed_p, node1, node2, strict, subpool)); @@ -3274,10 +2216,12 @@ x_get_file_delta_stream(svn_txdelta_stream_t **stream_p, apr_pool_t *scratch_pool = svn_pool_create(pool); if (source_root && source_path) - SVN_ERR(get_dag(&source_node, source_root, source_path, scratch_pool)); + SVN_ERR(svn_fs_x__get_dag_node(&source_node, source_root, source_path, + scratch_pool, scratch_pool)); else source_node = NULL; - SVN_ERR(get_dag(&target_node, target_root, target_path, scratch_pool)); + SVN_ERR(svn_fs_x__get_temp_dag_node(&target_node, target_root, target_path, + scratch_pool)); /* Create a delta stream that turns the source into the target. */ SVN_ERR(svn_fs_x__dag_get_file_delta_stream(stream_p, source_node, @@ -3292,91 +2236,142 @@ x_get_file_delta_stream(svn_txdelta_stream_t **stream_p, /* Finding Changes */ -/* Copy CHANGE into a FS API object allocated in RESULT_POOL and return - it in *RESULT_P. Pass CONTEXT to the ID API object being created. */ +/* Implement changes_iterator_vtable_t.get for in-txn change lists. + There is no specific FSAP data type, a simple APR hash iterator + to the underlying collection is sufficient. */ static svn_error_t * -construct_fs_path_change(svn_fs_path_change2_t **result_p, - svn_fs_x__id_context_t *context, - svn_fs_x__change_t *change, - apr_pool_t *result_pool) +x_txn_changes_iterator_get(svn_fs_path_change3_t **change, + svn_fs_path_change_iterator_t *iterator) +{ + apr_hash_index_t *hi = iterator->fsap_data; + + if (hi) + { + *change = apr_hash_this_val(hi); + iterator->fsap_data = apr_hash_next(hi); + } + else + { + *change = NULL; + } + + return SVN_NO_ERROR; +} + +static changes_iterator_vtable_t txn_changes_iterator_vtable = { - const svn_fs_id_t *id - = svn_fs_x__id_create(context, &change->noderev_id, result_pool); - svn_fs_path_change2_t *result - = svn_fs__path_change_create_internal(id, change->change_kind, - result_pool); + x_txn_changes_iterator_get +}; - result->text_mod = change->text_mod; - result->prop_mod = change->prop_mod; - result->node_kind = change->node_kind; +/* FSAP data structure for in-revision changes list iterators. */ +typedef struct fs_revision_changes_iterator_data_t +{ + /* Context that tells the lower layers from where to fetch the next + block of changes. */ + svn_fs_x__changes_context_t *context; - result->copyfrom_known = change->copyfrom_known; - result->copyfrom_rev = change->copyfrom_rev; - result->copyfrom_path = change->copyfrom_path; + /* Changes to send. */ + apr_array_header_t *changes; - result->mergeinfo_mod = change->mergeinfo_mod; + /* Current indexes within CHANGES. */ + int idx; - *result_p = result; + /* A cleanable scratch pool in case we need one. + No further sub-pool creation necessary. */ + apr_pool_t *scratch_pool; +} fs_revision_changes_iterator_data_t; + +static svn_error_t * +x_revision_changes_iterator_get(svn_fs_path_change3_t **change, + svn_fs_path_change_iterator_t *iterator) +{ + fs_revision_changes_iterator_data_t *data = iterator->fsap_data; + + /* If we exhausted our block of changes and did not reach the end of the + list, yet, fetch the next block. Note that that block may be empty. */ + if ((data->idx >= data->changes->nelts) && !data->context->eol) + { + apr_pool_t *changes_pool = data->changes->pool; + + /* Drop old changes block, read new block. */ + svn_pool_clear(changes_pool); + SVN_ERR(svn_fs_x__get_changes(&data->changes, data->context, + changes_pool, data->scratch_pool)); + data->idx = 0; + + /* Immediately release any temporary data. */ + svn_pool_clear(data->scratch_pool); + } + + if (data->idx < data->changes->nelts) + { + *change = APR_ARRAY_IDX(data->changes, data->idx, + svn_fs_x__change_t *); + ++data->idx; + } + else + { + *change = NULL; + } return SVN_NO_ERROR; } -/* Set *CHANGED_PATHS_P to a newly allocated hash containing - descriptions of the paths changed under ROOT. The hash is keyed - with const char * paths and has svn_fs_path_change2_t * values. Use - POOL for all allocations. */ -static svn_error_t * -x_paths_changed(apr_hash_t **changed_paths_p, - svn_fs_root_t *root, - apr_pool_t *pool) +static changes_iterator_vtable_t rev_changes_iterator_vtable = { - apr_hash_t *changed_paths; - svn_fs_path_change2_t *path_change; - svn_fs_x__id_context_t *context - = svn_fs_x__id_create_context(root->fs, pool); + x_revision_changes_iterator_get +}; +static svn_error_t * +x_report_changes(svn_fs_path_change_iterator_t **iterator, + svn_fs_root_t *root, + apr_pool_t *result_pool, + apr_pool_t *scratch_pool) +{ + svn_fs_path_change_iterator_t *result = apr_pcalloc(result_pool, + sizeof(*result)); if (root->is_txn_root) { - apr_hash_index_t *hi; + apr_hash_t *changed_paths; SVN_ERR(svn_fs_x__txn_changes_fetch(&changed_paths, root->fs, - root_txn_id(root), pool)); - for (hi = apr_hash_first(pool, changed_paths); - hi; - hi = apr_hash_next(hi)) - { - svn_fs_x__change_t *change = apr_hash_this_val(hi); - SVN_ERR(construct_fs_path_change(&path_change, context, change, - pool)); - apr_hash_set(changed_paths, - apr_hash_this_key(hi), apr_hash_this_key_len(hi), - path_change); - } + svn_fs_x__root_txn_id(root), + result_pool)); + + result->fsap_data = apr_hash_first(result_pool, changed_paths); + result->vtable = &txn_changes_iterator_vtable; } else { - apr_array_header_t *changes; - int i; + /* The block of changes that we retrieve need to live in a separately + cleanable pool. */ + apr_pool_t *changes_pool = svn_pool_create(result_pool); - SVN_ERR(svn_fs_x__get_changes(&changes, root->fs, root->rev, pool)); + /* Our iteration context info. */ + fs_revision_changes_iterator_data_t *data = apr_pcalloc(result_pool, + sizeof(*data)); - changed_paths = svn_hash__make(pool); - for (i = 0; i < changes->nelts; ++i) - { - svn_fs_x__change_t *change = APR_ARRAY_IDX(changes, i, - svn_fs_x__change_t *); - SVN_ERR(construct_fs_path_change(&path_change, context, change, - pool)); - apr_hash_set(changed_paths, change->path.data, change->path.len, - path_change); - } + /* This pool must remain valid as long as ITERATOR lives but will + be used only for temporary allocations and will be cleaned up + frequently. So, this must be a sub-pool of RESULT_POOL. */ + data->scratch_pool = svn_pool_create(result_pool); + + /* Fetch the first block of data. */ + SVN_ERR(svn_fs_x__create_changes_context(&data->context, + root->fs, root->rev, + result_pool, scratch_pool)); + SVN_ERR(svn_fs_x__get_changes(&data->changes, data->context, + changes_pool, scratch_pool)); + + /* Return the fully initialized object. */ + result->fsap_data = data; + result->vtable = &rev_changes_iterator_vtable; } - *changed_paths_p = changed_paths; + *iterator = result; return SVN_NO_ERROR; } - /* Our coolio opaque history object. */ typedef struct fs_history_data_t @@ -3394,6 +2389,15 @@ typedef struct fs_history_data_t /* FALSE until the first call to svn_fs_history_prev(). */ svn_boolean_t is_interesting; + + /* If not SVN_INVALID_REVISION, we know that the next copy operation + is at this revision. */ + svn_revnum_t next_copy; + + /* If used, see svn_fs_x__id_used, this is the noderev ID of + PATH@REVISION. */ + svn_fs_x__id_t current_id; + } fs_history_data_t; static svn_fs_history_t * @@ -3403,6 +2407,8 @@ assemble_history(svn_fs_t *fs, svn_boolean_t is_interesting, const char *path_hint, svn_revnum_t rev_hint, + svn_revnum_t next_copy, + const svn_fs_x__id_t *current_id, apr_pool_t *result_pool); @@ -3429,17 +2435,18 @@ x_node_history(svn_fs_history_t **history_p, /* Okay, all seems well. Build our history object and return it. */ *history_p = assemble_history(root->fs, path, root->rev, FALSE, NULL, - SVN_INVALID_REVNUM, result_pool); + SVN_INVALID_REVNUM, SVN_INVALID_REVNUM, + NULL, result_pool); return SVN_NO_ERROR; } -/* Find the youngest copyroot for path PARENT_PATH or its parents in +/* Find the youngest copyroot for path DAG_PATH or its parents in filesystem FS, and store the copyroot in *REV_P and *PATH_P. */ static svn_error_t * find_youngest_copyroot(svn_revnum_t *rev_p, const char **path_p, svn_fs_t *fs, - parent_path_t *parent_path) + svn_fs_x__dag_path_t *dag_path) { svn_revnum_t rev_mine; svn_revnum_t rev_parent = SVN_INVALID_REVNUM; @@ -3447,13 +2454,12 @@ find_youngest_copyroot(svn_revnum_t *rev_p, const char *path_parent = NULL; /* First find our parent's youngest copyroot. */ - if (parent_path->parent) + if (dag_path->parent) SVN_ERR(find_youngest_copyroot(&rev_parent, &path_parent, fs, - parent_path->parent)); + dag_path->parent)); /* Find our copyroot. */ - SVN_ERR(svn_fs_x__dag_get_copyroot(&rev_mine, &path_mine, - parent_path->node)); + svn_fs_x__dag_get_copyroot(&rev_mine, &path_mine, dag_path->node); /* If a parent and child were copied to in the same revision, prefer the child copy target, since it is the copy relevant to the @@ -3481,26 +2487,25 @@ x_closest_copy(svn_fs_root_t **root_p, apr_pool_t *pool) { svn_fs_t *fs = root->fs; - parent_path_t *parent_path, *copy_dst_parent_path; + svn_fs_x__dag_path_t *dag_path, *copy_dst_dag_path; svn_revnum_t copy_dst_rev, created_rev; const char *copy_dst_path; svn_fs_root_t *copy_dst_root; dag_node_t *copy_dst_node; - svn_boolean_t related; apr_pool_t *scratch_pool = svn_pool_create(pool); /* Initialize return values. */ *root_p = NULL; *path_p = NULL; - path = svn_fs__canonicalize_abspath(path, scratch_pool); - SVN_ERR(open_path(&parent_path, root, path, 0, FALSE, scratch_pool)); + SVN_ERR(svn_fs_x__get_dag_path(&dag_path, root, path, 0, FALSE, + scratch_pool, scratch_pool)); /* Find the youngest copyroot in the path of this node-rev, which will indicate the target of the innermost copy affecting the node-rev. */ SVN_ERR(find_youngest_copyroot(©_dst_rev, ©_dst_path, - fs, parent_path)); + fs, dag_path)); if (copy_dst_rev == 0) /* There are no copies affecting this node-rev. */ { svn_pool_destroy(scratch_pool); @@ -3511,19 +2516,17 @@ x_closest_copy(svn_fs_root_t **root_p, revision between COPY_DST_REV and REV. Make sure that PATH exists as of COPY_DST_REV and is related to this node-rev. */ SVN_ERR(svn_fs_x__revision_root(©_dst_root, fs, copy_dst_rev, pool)); - SVN_ERR(open_path(©_dst_parent_path, copy_dst_root, path, - open_path_node_only | open_path_allow_null, FALSE, - scratch_pool)); - if (copy_dst_parent_path == NULL) + SVN_ERR(svn_fs_x__get_dag_path(©_dst_dag_path, copy_dst_root, path, + svn_fs_x__dag_path_allow_null, FALSE, + scratch_pool, scratch_pool)); + if (copy_dst_dag_path == NULL) { svn_pool_destroy(scratch_pool); return SVN_NO_ERROR; } - copy_dst_node = copy_dst_parent_path->node; - SVN_ERR(svn_fs_x__dag_related_node(&related, copy_dst_node, - parent_path->node)); - if (!related) + copy_dst_node = copy_dst_dag_path->node; + if (!svn_fs_x__dag_related_node(copy_dst_node, dag_path->node)) { svn_pool_destroy(scratch_pool); return SVN_NO_ERROR; @@ -3545,15 +2548,11 @@ x_closest_copy(svn_fs_root_t **root_p, */ created_rev = svn_fs_x__dag_get_revision(copy_dst_node); if (created_rev == copy_dst_rev) - { - svn_fs_x__id_t pred; - SVN_ERR(svn_fs_x__dag_get_predecessor_id(&pred, copy_dst_node)); - if (!svn_fs_x__id_used(&pred)) - { - svn_pool_destroy(scratch_pool); - return SVN_NO_ERROR; - } - } + if (!svn_fs_x__id_used(svn_fs_x__dag_get_predecessor_id(copy_dst_node))) + { + svn_pool_destroy(scratch_pool); + return SVN_NO_ERROR; + } /* The copy destination checks out. Return it. */ *root_p = copy_dst_root; @@ -3573,10 +2572,8 @@ x_node_origin_rev(svn_revnum_t *revision, svn_fs_x__id_t node_id; dag_node_t *node; - path = svn_fs__canonicalize_abspath(path, scratch_pool); - - SVN_ERR(get_dag(&node, root, path, scratch_pool)); - SVN_ERR(svn_fs_x__dag_get_node_id(&node_id, node)); + SVN_ERR(svn_fs_x__get_temp_dag_node(&node, root, path, scratch_pool)); + node_id = *svn_fs_x__dag_get_node_id(node); *revision = svn_fs_x__get_revnum(node_id.change_set); @@ -3596,16 +2593,56 @@ history_prev(svn_fs_history_t **prev_history, svn_revnum_t commit_rev, src_rev, dst_rev; svn_revnum_t revision = fhd->revision; svn_fs_t *fs = fhd->fs; - parent_path_t *parent_path; + svn_fs_x__dag_path_t *dag_path; dag_node_t *node; svn_fs_root_t *root; svn_boolean_t reported = fhd->is_interesting; svn_revnum_t copyroot_rev; const char *copyroot_path; + svn_fs_x__id_t pred_id; /* Initialize our return value. */ *prev_history = NULL; + /* When following history, there tend to be long sections of linear + history where there are no copies at PATH or its parents. Within + these sections, we only need to follow the node history. */ + if ( SVN_IS_VALID_REVNUM(fhd->next_copy) + && revision > fhd->next_copy + && svn_fs_x__id_used(&fhd->current_id)) + { + /* We know the last reported node (CURRENT_ID) and the NEXT_COPY + revision is somewhat further in the past. */ + svn_fs_x__noderev_t *noderev; + assert(reported); + + /* Get the previous node change. If there is none, then we already + reported the initial addition and this history traversal is done. */ + SVN_ERR(svn_fs_x__get_node_revision(&noderev, fs, &fhd->current_id, + scratch_pool, scratch_pool)); + if (! svn_fs_x__id_used(&noderev->predecessor_id)) + return SVN_NO_ERROR; + + /* If the previous node change is younger than the next copy, it is + part of the linear history section. */ + commit_rev = svn_fs_x__get_revnum(noderev->predecessor_id.change_set); + if (commit_rev > fhd->next_copy) + { + /* Within the linear history, simply report all node changes and + continue with the respective predecessor. */ + *prev_history = assemble_history(fs, noderev->created_path, + commit_rev, TRUE, NULL, + SVN_INVALID_REVNUM, + fhd->next_copy, + &noderev->predecessor_id, + result_pool); + + return SVN_NO_ERROR; + } + + /* We hit a copy. Fall back to the standard code path. */ + } + /* If our last history report left us hints about where to pickup the chase, then our last report was on the destination of a copy. If we are crossing copies, start from those locations, @@ -3624,10 +2661,12 @@ history_prev(svn_fs_history_t **prev_history, /* Open PATH/REVISION, and get its node and a bunch of other goodies. */ - SVN_ERR(open_path(&parent_path, root, path, 0, FALSE, scratch_pool)); - node = parent_path->node; + SVN_ERR(svn_fs_x__get_dag_path(&dag_path, root, path, 0, FALSE, + scratch_pool, scratch_pool)); + node = dag_path->node; commit_path = svn_fs_x__dag_get_created_path(node); commit_rev = svn_fs_x__dag_get_revision(node); + svn_fs_x__id_reset(&pred_id); /* The Subversion filesystem is written in such a way that a given line of history may have at most one interesting history point @@ -3644,7 +2683,9 @@ history_prev(svn_fs_history_t **prev_history, need now to do so) ... */ *prev_history = assemble_history(fs, commit_path, commit_rev, TRUE, NULL, - SVN_INVALID_REVNUM, result_pool); + SVN_INVALID_REVNUM, + SVN_INVALID_REVNUM, NULL, + result_pool); return SVN_NO_ERROR; } else @@ -3652,9 +2693,7 @@ history_prev(svn_fs_history_t **prev_history, /* ... or we *have* reported on this revision, and must now progress toward this node's predecessor (unless there is no predecessor, in which case we're all done!). */ - svn_fs_x__id_t pred_id; - - SVN_ERR(svn_fs_x__dag_get_predecessor_id(&pred_id, node)); + pred_id = *svn_fs_x__dag_get_predecessor_id(node); if (!svn_fs_x__id_used(&pred_id)) return SVN_NO_ERROR; @@ -3670,7 +2709,7 @@ history_prev(svn_fs_history_t **prev_history, /* Find the youngest copyroot in the path of this node, including itself. */ SVN_ERR(find_youngest_copyroot(©root_rev, ©root_path, fs, - parent_path)); + dag_path)); /* Initialize some state variables. */ src_path = NULL; @@ -3685,7 +2724,8 @@ history_prev(svn_fs_history_t **prev_history, SVN_ERR(svn_fs_x__revision_root(©root_root, fs, copyroot_rev, scratch_pool)); - SVN_ERR(get_dag(&node, copyroot_root, copyroot_path, scratch_pool)); + SVN_ERR(svn_fs_x__get_temp_dag_node(&node, copyroot_root, + copyroot_path, scratch_pool)); copy_dst = svn_fs_x__dag_get_created_path(node); /* If our current path was the very destination of the copy, @@ -3703,8 +2743,8 @@ history_prev(svn_fs_history_t **prev_history, /* If we get here, then our current path is the destination of, or the child of the destination of, a copy. Fill in the return values and get outta here. */ - SVN_ERR(svn_fs_x__dag_get_copyfrom_rev(&src_rev, node)); - SVN_ERR(svn_fs_x__dag_get_copyfrom_path(©_src, node)); + src_rev = svn_fs_x__dag_get_copyfrom_rev(node); + copy_src = svn_fs_x__dag_get_copyfrom_path(node); dst_rev = copyroot_rev; src_path = svn_fspath__join(copy_src, remainder_path, scratch_pool); @@ -3725,12 +2765,18 @@ history_prev(svn_fs_history_t **prev_history, retry = TRUE; *prev_history = assemble_history(fs, path, dst_rev, ! retry, - src_path, src_rev, result_pool); + src_path, src_rev, + SVN_INVALID_REVNUM, NULL, + result_pool); } else { + /* We know the next copy revision. If we are not at the copy rev + itself, we will also know the predecessor node ID and the next + invocation will use the optimized "linear history" code path. */ *prev_history = assemble_history(fs, commit_path, commit_rev, TRUE, - NULL, SVN_INVALID_REVNUM, result_pool); + NULL, SVN_INVALID_REVNUM, + copyroot_rev, &pred_id, result_pool); } return SVN_NO_ERROR; @@ -3761,10 +2807,12 @@ fs_history_prev(svn_fs_history_t **prev_history_p, if (! fhd->is_interesting) prev_history = assemble_history(fs, "/", fhd->revision, 1, NULL, SVN_INVALID_REVNUM, + SVN_INVALID_REVNUM, NULL, result_pool); else if (fhd->revision > 0) prev_history = assemble_history(fs, "/", fhd->revision - 1, 1, NULL, SVN_INVALID_REVNUM, + SVN_INVALID_REVNUM, NULL, result_pool); } else @@ -3824,6 +2872,8 @@ assemble_history(svn_fs_t *fs, svn_boolean_t is_interesting, const char *path_hint, svn_revnum_t rev_hint, + svn_revnum_t next_copy, + const svn_fs_x__id_t *current_id, apr_pool_t *result_pool) { svn_fs_history_t *history = apr_pcalloc(result_pool, sizeof(*history)); @@ -3835,8 +2885,14 @@ assemble_history(svn_fs_t *fs, ? svn_fs__canonicalize_abspath(path_hint, result_pool) : NULL; fhd->rev_hint = rev_hint; + fhd->next_copy = next_copy; fhd->fs = fs; + if (current_id) + fhd->current_id = *current_id; + else + svn_fs_x__id_reset(&fhd->current_id); + history->vtable = &history_vtable; history->fsap_data = fhd; return history; @@ -3848,21 +2904,19 @@ assemble_history(svn_fs_t *fs, /* DIR_DAG is a directory DAG node which has mergeinfo in its descendants. This function iterates over its children. For each - child with immediate mergeinfo, it adds its mergeinfo to - RESULT_CATALOG. appropriate arguments. For each child with - descendants with mergeinfo, it recurses. Note that it does *not* - call the action on the path for DIR_DAG itself. - - POOL is used for temporary allocations, including the mergeinfo - hashes passed to actions; RESULT_POOL is used for the mergeinfo added - to RESULT_CATALOG. + child with immediate mergeinfo, call RECEIVER with it and BATON. + For each child with descendants with mergeinfo, it recurses. Note + that it does *not* call the action on the path for DIR_DAG itself. + + SCRATCH_POOL is used for temporary allocations, including the mergeinfo + hashes passed to actions. */ static svn_error_t * crawl_directory_dag_for_mergeinfo(svn_fs_root_t *root, const char *this_path, dag_node_t *dir_dag, - svn_mergeinfo_catalog_t result_catalog, - apr_pool_t *result_pool, + svn_fs_mergeinfo_receiver_t receiver, + void *baton, apr_pool_t *scratch_pool) { apr_array_header_t *entries; @@ -3873,20 +2927,18 @@ crawl_directory_dag_for_mergeinfo(svn_fs_root_t *root, iterpool)); for (i = 0; i < entries->nelts; ++i) { - svn_fs_x__dirent_t *dirent = APR_ARRAY_IDX(entries, i, svn_fs_x__dirent_t *); + svn_fs_x__dirent_t *dirent + = APR_ARRAY_IDX(entries, i, svn_fs_x__dirent_t *); const char *kid_path; dag_node_t *kid_dag; - svn_boolean_t has_mergeinfo, go_down; svn_pool_clear(iterpool); kid_path = svn_fspath__join(this_path, dirent->name, iterpool); - SVN_ERR(get_dag(&kid_dag, root, kid_path, iterpool)); + SVN_ERR(svn_fs_x__get_temp_dag_node(&kid_dag, root, kid_path, + iterpool)); - SVN_ERR(svn_fs_x__dag_has_mergeinfo(&has_mergeinfo, kid_dag)); - SVN_ERR(svn_fs_x__dag_has_descendants_with_mergeinfo(&go_down, kid_dag)); - - if (has_mergeinfo) + if (svn_fs_x__dag_has_mergeinfo(kid_dag)) { /* Save this particular node's mergeinfo. */ apr_hash_t *proplist; @@ -3912,7 +2964,7 @@ crawl_directory_dag_for_mergeinfo(svn_fs_root_t *root, error. */ err = svn_mergeinfo_parse(&kid_mergeinfo, mergeinfo_string->data, - result_pool); + iterpool); if (err) { if (err->apr_err == SVN_ERR_MERGEINFO_PARSE_ERROR) @@ -3922,17 +2974,16 @@ crawl_directory_dag_for_mergeinfo(svn_fs_root_t *root, } else { - svn_hash_sets(result_catalog, apr_pstrdup(result_pool, kid_path), - kid_mergeinfo); + SVN_ERR(receiver(kid_path, kid_mergeinfo, baton, iterpool)); } } - if (go_down) + if (svn_fs_x__dag_has_descendants_with_mergeinfo(kid_dag)) SVN_ERR(crawl_directory_dag_for_mergeinfo(root, kid_path, kid_dag, - result_catalog, - result_pool, + receiver, + baton, iterpool)); } @@ -3940,62 +2991,39 @@ crawl_directory_dag_for_mergeinfo(svn_fs_root_t *root, return SVN_NO_ERROR; } -/* Return the cache key as a combination of REV_ROOT->REV, the inheritance - flags INHERIT and ADJUST_INHERITED_MERGEINFO, and the PATH. The result - will be allocated in RESULT_POOL. - */ -static const char * -mergeinfo_cache_key(const char *path, - svn_fs_root_t *rev_root, - svn_mergeinfo_inheritance_t inherit, - svn_boolean_t adjust_inherited_mergeinfo, - apr_pool_t *result_pool) -{ - apr_int64_t number = rev_root->rev; - number = number * 4 - + (inherit == svn_mergeinfo_nearest_ancestor ? 2 : 0) - + (adjust_inherited_mergeinfo ? 1 : 0); - - return svn_fs_x__combine_number_and_string(number, path, result_pool); -} - /* Calculates the mergeinfo for PATH under REV_ROOT using inheritance type INHERIT. Returns it in *MERGEINFO, or NULL if there is none. The result is allocated in RESULT_POOL; SCRATCH_POOL is used for temporary allocations. */ static svn_error_t * -get_mergeinfo_for_path_internal(svn_mergeinfo_t *mergeinfo, - svn_fs_root_t *rev_root, - const char *path, - svn_mergeinfo_inheritance_t inherit, - svn_boolean_t adjust_inherited_mergeinfo, - apr_pool_t *result_pool, - apr_pool_t *scratch_pool) +get_mergeinfo_for_path(svn_mergeinfo_t *mergeinfo, + svn_fs_root_t *rev_root, + const char *path, + svn_mergeinfo_inheritance_t inherit, + svn_boolean_t adjust_inherited_mergeinfo, + apr_pool_t *result_pool, + apr_pool_t *scratch_pool) { - parent_path_t *parent_path, *nearest_ancestor; + svn_fs_x__dag_path_t *dag_path, *nearest_ancestor; apr_hash_t *proplist; svn_string_t *mergeinfo_string; - path = svn_fs__canonicalize_abspath(path, scratch_pool); - - SVN_ERR(open_path(&parent_path, rev_root, path, 0, FALSE, scratch_pool)); + *mergeinfo = NULL; + SVN_ERR(svn_fs_x__get_dag_path(&dag_path, rev_root, path, 0, FALSE, + scratch_pool, scratch_pool)); - if (inherit == svn_mergeinfo_nearest_ancestor && ! parent_path->parent) + if (inherit == svn_mergeinfo_nearest_ancestor && ! dag_path->parent) return SVN_NO_ERROR; if (inherit == svn_mergeinfo_nearest_ancestor) - nearest_ancestor = parent_path->parent; + nearest_ancestor = dag_path->parent; else - nearest_ancestor = parent_path; + nearest_ancestor = dag_path; while (TRUE) { - svn_boolean_t has_mergeinfo; - - SVN_ERR(svn_fs_x__dag_has_mergeinfo(&has_mergeinfo, - nearest_ancestor->node)); - if (has_mergeinfo) + if (svn_fs_x__dag_has_mergeinfo(nearest_ancestor->node)) break; /* No need to loop if we're looking for explicit mergeinfo. */ @@ -4046,7 +3074,7 @@ get_mergeinfo_for_path_internal(svn_mergeinfo_t *mergeinfo, can return the mergeinfo results directly. Otherwise, we're inheriting the mergeinfo, so we need to a) remove non-inheritable ranges and b) telescope the merged-from paths. */ - if (adjust_inherited_mergeinfo && (nearest_ancestor != parent_path)) + if (adjust_inherited_mergeinfo && (nearest_ancestor != dag_path)) { svn_mergeinfo_t tmp_mergeinfo; @@ -4056,7 +3084,7 @@ get_mergeinfo_for_path_internal(svn_mergeinfo_t *mergeinfo, scratch_pool, scratch_pool)); SVN_ERR(svn_fs__append_to_merged_froms(mergeinfo, tmp_mergeinfo, parent_path_relpath( - parent_path, nearest_ancestor, + dag_path, nearest_ancestor, scratch_pool), result_pool)); } @@ -4064,99 +3092,44 @@ get_mergeinfo_for_path_internal(svn_mergeinfo_t *mergeinfo, return SVN_NO_ERROR; } -/* Caching wrapper around get_mergeinfo_for_path_internal(). - */ +/* Invoke RECEIVER with BATON for each mergeinfo found on descendants of + PATH (but not PATH itself). Use SCRATCH_POOL for temporary values. */ static svn_error_t * -get_mergeinfo_for_path(svn_mergeinfo_t *mergeinfo, - svn_fs_root_t *rev_root, - const char *path, - svn_mergeinfo_inheritance_t inherit, - svn_boolean_t adjust_inherited_mergeinfo, - apr_pool_t *result_pool, - apr_pool_t *scratch_pool) -{ - svn_fs_x__data_t *ffd = rev_root->fs->fsap_data; - const char *cache_key; - svn_boolean_t found = FALSE; - svn_stringbuf_t *mergeinfo_exists; - - *mergeinfo = NULL; - - cache_key = mergeinfo_cache_key(path, rev_root, inherit, - adjust_inherited_mergeinfo, scratch_pool); - if (ffd->mergeinfo_existence_cache) - { - SVN_ERR(svn_cache__get((void **)&mergeinfo_exists, &found, - ffd->mergeinfo_existence_cache, - cache_key, result_pool)); - if (found && mergeinfo_exists->data[0] == '1') - SVN_ERR(svn_cache__get((void **)mergeinfo, &found, - ffd->mergeinfo_cache, - cache_key, result_pool)); - } - - if (! found) - { - SVN_ERR(get_mergeinfo_for_path_internal(mergeinfo, rev_root, path, - inherit, - adjust_inherited_mergeinfo, - result_pool, scratch_pool)); - if (ffd->mergeinfo_existence_cache) - { - mergeinfo_exists = svn_stringbuf_create(*mergeinfo ? "1" : "0", - scratch_pool); - SVN_ERR(svn_cache__set(ffd->mergeinfo_existence_cache, - cache_key, mergeinfo_exists, scratch_pool)); - if (*mergeinfo) - SVN_ERR(svn_cache__set(ffd->mergeinfo_cache, - cache_key, *mergeinfo, scratch_pool)); - } - } - - return SVN_NO_ERROR; -} - -/* Adds mergeinfo for each descendant of PATH (but not PATH itself) - under ROOT to RESULT_CATALOG. Returned values are allocated in - RESULT_POOL; temporary values in POOL. */ -static svn_error_t * -add_descendant_mergeinfo(svn_mergeinfo_catalog_t result_catalog, - svn_fs_root_t *root, +add_descendant_mergeinfo(svn_fs_root_t *root, const char *path, - apr_pool_t *result_pool, + svn_fs_mergeinfo_receiver_t receiver, + void *baton, apr_pool_t *scratch_pool) { dag_node_t *this_dag; - svn_boolean_t go_down; - SVN_ERR(get_dag(&this_dag, root, path, scratch_pool)); - SVN_ERR(svn_fs_x__dag_has_descendants_with_mergeinfo(&go_down, - this_dag)); - if (go_down) + SVN_ERR(svn_fs_x__get_temp_dag_node(&this_dag, root, path, scratch_pool)); + if (svn_fs_x__dag_has_descendants_with_mergeinfo(this_dag)) SVN_ERR(crawl_directory_dag_for_mergeinfo(root, path, this_dag, - result_catalog, - result_pool, + receiver, + baton, scratch_pool)); return SVN_NO_ERROR; } -/* Get the mergeinfo for a set of paths, returned in - *MERGEINFO_CATALOG. Returned values are allocated in - POOL, while temporary values are allocated in a sub-pool. */ +/* Find all the mergeinfo for a set of PATHS under ROOT and report it + through RECEIVER with BATON. INHERITED, INCLUDE_DESCENDANTS and + ADJUST_INHERITED_MERGEINFO are the same as in the FS API. + + Allocate temporary values are allocated in SCRATCH_POOL. */ static svn_error_t * get_mergeinfos_for_paths(svn_fs_root_t *root, - svn_mergeinfo_catalog_t *mergeinfo_catalog, const apr_array_header_t *paths, svn_mergeinfo_inheritance_t inherit, svn_boolean_t include_descendants, svn_boolean_t adjust_inherited_mergeinfo, - apr_pool_t *result_pool, + svn_fs_mergeinfo_receiver_t receiver, + void *baton, apr_pool_t *scratch_pool) { - svn_mergeinfo_catalog_t result_catalog = svn_hash__make(result_pool); apr_pool_t *iterpool = svn_pool_create(scratch_pool); int i; @@ -4170,7 +3143,7 @@ get_mergeinfos_for_paths(svn_fs_root_t *root, err = get_mergeinfo_for_path(&path_mergeinfo, root, path, inherit, adjust_inherited_mergeinfo, - result_pool, iterpool); + iterpool, iterpool); if (err) { if (err->apr_err == SVN_ERR_MERGEINFO_PARSE_ERROR) @@ -4186,27 +3159,26 @@ get_mergeinfos_for_paths(svn_fs_root_t *root, } if (path_mergeinfo) - svn_hash_sets(result_catalog, path, path_mergeinfo); + SVN_ERR(receiver(path, path_mergeinfo, baton, iterpool)); if (include_descendants) - SVN_ERR(add_descendant_mergeinfo(result_catalog, root, path, - result_pool, scratch_pool)); + SVN_ERR(add_descendant_mergeinfo(root, path, receiver, baton, + iterpool)); } svn_pool_destroy(iterpool); - *mergeinfo_catalog = result_catalog; return SVN_NO_ERROR; } /* Implements svn_fs_get_mergeinfo. */ static svn_error_t * -x_get_mergeinfo(svn_mergeinfo_catalog_t *catalog, - svn_fs_root_t *root, +x_get_mergeinfo(svn_fs_root_t *root, const apr_array_header_t *paths, svn_mergeinfo_inheritance_t inherit, svn_boolean_t include_descendants, svn_boolean_t adjust_inherited_mergeinfo, - apr_pool_t *result_pool, + svn_fs_mergeinfo_receiver_t receiver, + void *baton, apr_pool_t *scratch_pool) { /* We require a revision root. */ @@ -4214,17 +3186,18 @@ x_get_mergeinfo(svn_mergeinfo_catalog_t *catalog, return svn_error_create(SVN_ERR_FS_NOT_REVISION_ROOT, NULL, NULL); /* Retrieve a path -> mergeinfo hash mapping. */ - return get_mergeinfos_for_paths(root, catalog, paths, - inherit, + return get_mergeinfos_for_paths(root, paths, inherit, include_descendants, adjust_inherited_mergeinfo, - result_pool, scratch_pool); + receiver, baton, + scratch_pool); } /* The vtable associated with root objects. */ static root_vtable_t root_vtable = { - x_paths_changed, + NULL, + x_report_changes, svn_fs_x__check_path, x_node_history, x_node_id, @@ -4308,20 +3281,6 @@ make_txn_root(svn_fs_root_t **root_p, root->txn = svn_fs_x__txn_name(txn_id, root->pool); root->txn_flags = flags; root->rev = base_rev; - - /* Because this cache actually tries to invalidate elements, keep - the number of elements per page down. - - Note that since dag_node_cache_invalidate uses svn_cache__iter, - this *cannot* be a memcache-based cache. */ - SVN_ERR(svn_cache__create_inprocess(&(frd->txn_node_cache), - svn_fs_x__dag_serialize, - svn_fs_x__dag_deserialize, - APR_HASH_KEY_STRING, - 32, 20, FALSE, - root->txn, - root->pool)); - root->fsap_data = frd; *root_p = root; @@ -4370,10 +3329,10 @@ verify_node(dag_node_t *node, } /* Fetch some data. */ - SVN_ERR(svn_fs_x__dag_has_mergeinfo(&has_mergeinfo, node)); - SVN_ERR(svn_fs_x__dag_get_mergeinfo_count(&mergeinfo_count, node)); - SVN_ERR(svn_fs_x__dag_get_predecessor_id(&pred_id, node)); - SVN_ERR(svn_fs_x__dag_get_predecessor_count(&pred_count, node)); + has_mergeinfo = svn_fs_x__dag_has_mergeinfo(node); + mergeinfo_count = svn_fs_x__dag_get_mergeinfo_count(node); + pred_id = *svn_fs_x__dag_get_predecessor_id(node); + pred_count = svn_fs_x__dag_get_predecessor_count(node); kind = svn_fs_x__dag_node_kind(node); /* Sanity check. */ @@ -4390,7 +3349,7 @@ verify_node(dag_node_t *node, int pred_pred_count; SVN_ERR(svn_fs_x__dag_get_node(&pred, fs, &pred_id, iterpool, iterpool)); - SVN_ERR(svn_fs_x__dag_get_predecessor_count(&pred_pred_count, pred)); + pred_pred_count = svn_fs_x__dag_get_predecessor_count(pred); if (pred_pred_count+1 != pred_count) return svn_error_createf(SVN_ERR_FS_CORRUPT, NULL, "Predecessor count mismatch: " @@ -4442,8 +3401,7 @@ verify_node(dag_node_t *node, SVN_ERR(svn_fs_x__dag_get_node(&child, fs, &dirent->id, iterpool, iterpool)); SVN_ERR(verify_node(child, rev, parent_nodes, iterpool)); - SVN_ERR(svn_fs_x__dag_get_mergeinfo_count(&child_mergeinfo, - child)); + child_mergeinfo = svn_fs_x__dag_get_mergeinfo_count(child); } else { @@ -4490,7 +3448,9 @@ svn_fs_x__verify_root(svn_fs_root_t *root, When this code is called in the library, we want to ensure we use the on-disk data --- rather than some data that was read in the possibly-distance past and cached since. */ - SVN_ERR(root_node(&root_dir, root, scratch_pool, scratch_pool)); + SVN_ERR(svn_fs_x__dag_root(&root_dir, root->fs, + svn_fs_x__root_change_set(root), + scratch_pool, scratch_pool)); /* Recursively verify ROOT_DIR. */ parent_nodes = apr_array_make(scratch_pool, 16, sizeof(dag_node_t *)); @@ -4502,7 +3462,7 @@ svn_fs_x__verify_root(svn_fs_root_t *root, svn_boolean_t has_predecessor; /* Only r0 should have no predecessor. */ - SVN_ERR(svn_fs_x__dag_get_predecessor_id(&pred_id, root_dir)); + pred_id = *svn_fs_x__dag_get_predecessor_id(root_dir); has_predecessor = svn_fs_x__id_used(&pred_id); if (!root->is_txn_root && has_predecessor != !!root->rev) return svn_error_createf(SVN_ERR_FS_CORRUPT, NULL, diff --git a/subversion/libsvn_fs_x/tree.h b/subversion/libsvn_fs_x/tree.h index 9c5d44aba842..58ad0dadabbb 100644 --- a/subversion/libsvn_fs_x/tree.h +++ b/subversion/libsvn_fs_x/tree.h @@ -20,10 +20,11 @@ * ==================================================================== */ -#ifndef SVN_LIBSVN_FS_TREE_H -#define SVN_LIBSVN_FS_TREE_H +#ifndef SVN_LIBSVN_FS_X_TREE_H +#define SVN_LIBSVN_FS_X_TREE_H #include "fs.h" +#include "dag.h" #ifdef __cplusplus extern "C" { @@ -31,9 +32,13 @@ extern "C" { -/* In RESULT_POOL, create an instance of a DAG node 1st level cache. */ -svn_fs_x__dag_cache_t* -svn_fs_x__create_dag_cache(apr_pool_t *result_pool); +/* Return the transaction ID to a given transaction ROOT. */ +svn_fs_x__txn_id_t +svn_fs_x__root_txn_id(svn_fs_root_t *root); + +/* Return the change set to a given ROOT. */ +svn_fs_x__change_set_t +svn_fs_x__root_change_set(svn_fs_root_t *root); /* Set *ROOT_P to the root directory of revision REV in filesystem FS. Allocate the structure in POOL. */ @@ -109,4 +114,4 @@ svn_fs_x__info_config_files(apr_array_header_t **files, } #endif /* __cplusplus */ -#endif /* SVN_LIBSVN_FS_TREE_H */ +#endif /* SVN_LIBSVN_FS_X_TREE_H */ diff --git a/subversion/libsvn_fs_x/util.c b/subversion/libsvn_fs_x/util.c index da004ad8febb..bab47fc6d01c 100644 --- a/subversion/libsvn_fs_x/util.c +++ b/subversion/libsvn_fs_x/util.c @@ -110,11 +110,17 @@ svn_fs_x__path_current(svn_fs_t *fs, } const char * +svn_fs_x__path_next(svn_fs_t *fs, + apr_pool_t *result_pool) +{ + return svn_dirent_join(fs->path, PATH_NEXT, result_pool); +} + +const char * svn_fs_x__path_txn_current(svn_fs_t *fs, apr_pool_t *result_pool) { - return svn_dirent_join(fs->path, PATH_TXN_CURRENT, - result_pool); + return svn_dirent_join(fs->path, PATH_TXN_CURRENT, result_pool); } const char * @@ -147,14 +153,12 @@ svn_fs_x__path_revprop_generation(svn_fs_t *fs, /* Return the full path of the file FILENAME within revision REV's shard in * FS. If FILENAME is NULL, return the shard directory directory itself. - * REVPROPS indicates the parent of the shard parent folder ("revprops" or - * "revs"). PACKED says whether we want the packed shard's name. + * PACKED says whether we want the packed shard's name. * * Allocate the result in RESULT_POOL. */static const char* construct_shard_sub_path(svn_fs_t *fs, svn_revnum_t rev, - svn_boolean_t revprops, svn_boolean_t packed, const char *filename, apr_pool_t *result_pool) @@ -162,9 +166,6 @@ construct_shard_sub_path(svn_fs_t *fs, svn_fs_x__data_t *ffd = fs->fsap_data; char buffer[SVN_INT64_BUFFER_SIZE + sizeof(PATH_EXT_PACKED_SHARD)] = { 0 }; - /* Select the appropriate parent path constant. */ - const char *parent = revprops ? PATH_REVPROPS_DIR : PATH_REVS_DIR; - /* String containing the shard number. */ apr_size_t len = svn__i64toa(buffer, rev / ffd->max_files_per_dir); @@ -173,7 +174,7 @@ construct_shard_sub_path(svn_fs_t *fs, strncpy(buffer + len, PATH_EXT_PACKED_SHARD, sizeof(buffer) - len - 1); /* This will also work for NULL FILENAME as well. */ - return svn_dirent_join_many(result_pool, fs->path, parent, buffer, + return svn_dirent_join_many(result_pool, fs->path, PATH_REVS_DIR, buffer, filename, SVN_VA_NULL); } @@ -184,15 +185,15 @@ svn_fs_x__path_rev_packed(svn_fs_t *fs, apr_pool_t *result_pool) { assert(svn_fs_x__is_packed_rev(fs, rev)); - return construct_shard_sub_path(fs, rev, FALSE, TRUE, kind, result_pool); + return construct_shard_sub_path(fs, rev, TRUE, kind, result_pool); } const char * -svn_fs_x__path_rev_shard(svn_fs_t *fs, - svn_revnum_t rev, - apr_pool_t *result_pool) +svn_fs_x__path_shard(svn_fs_t *fs, + svn_revnum_t rev, + apr_pool_t *result_pool) { - return construct_shard_sub_path(fs, rev, FALSE, FALSE, NULL, result_pool); + return construct_shard_sub_path(fs, rev, FALSE, NULL, result_pool); } const char * @@ -200,11 +201,12 @@ svn_fs_x__path_rev(svn_fs_t *fs, svn_revnum_t rev, apr_pool_t *result_pool) { - char buffer[SVN_INT64_BUFFER_SIZE]; - svn__i64toa(buffer, rev); + char buffer[SVN_INT64_BUFFER_SIZE + 1]; + buffer[0] = 'r'; + svn__i64toa(buffer + 1, rev); assert(! svn_fs_x__is_packed_rev(fs, rev)); - return construct_shard_sub_path(fs, rev, FALSE, FALSE, buffer, result_pool); + return construct_shard_sub_path(fs, rev, FALSE, buffer, result_pool); } const char * @@ -218,19 +220,11 @@ svn_fs_x__path_rev_absolute(svn_fs_t *fs, } const char * -svn_fs_x__path_revprops_shard(svn_fs_t *fs, - svn_revnum_t rev, - apr_pool_t *result_pool) -{ - return construct_shard_sub_path(fs, rev, TRUE, FALSE, NULL, result_pool); -} - -const char * -svn_fs_x__path_revprops_pack_shard(svn_fs_t *fs, +svn_fs_x__path_pack_shard(svn_fs_t *fs, svn_revnum_t rev, apr_pool_t *result_pool) { - return construct_shard_sub_path(fs, rev, TRUE, TRUE, NULL, result_pool); + return construct_shard_sub_path(fs, rev, TRUE, NULL, result_pool); } const char * @@ -238,11 +232,17 @@ svn_fs_x__path_revprops(svn_fs_t *fs, svn_revnum_t rev, apr_pool_t *result_pool) { - char buffer[SVN_INT64_BUFFER_SIZE]; - svn__i64toa(buffer, rev); + char buffer[SVN_INT64_BUFFER_SIZE + 1]; + buffer[0] = 'p'; + svn__i64toa(buffer + 1, rev); assert(! svn_fs_x__is_packed_revprop(fs, rev)); - return construct_shard_sub_path(fs, rev, TRUE, FALSE, buffer, result_pool); + + /* Revprops for packed r0 are not packed, yet stored in the packed shard. + Hence, the second flag must check for packed _rev_ - not revprop. */ + return construct_shard_sub_path(fs, rev, + svn_fs_x__is_packed_rev(fs, rev) /* sic! */, + buffer, result_pool); } const char * @@ -340,14 +340,6 @@ svn_fs_x__path_txn_props(svn_fs_t *fs, return construct_txn_path(fs, txn_id, PATH_TXN_PROPS, result_pool); } -const char * -svn_fs_x__path_txn_props_final(svn_fs_t *fs, - svn_fs_x__txn_id_t txn_id, - apr_pool_t *result_pool) -{ - return construct_txn_path(fs, txn_id, PATH_TXN_PROPS_FINAL, result_pool); -} - const char* svn_fs_x__path_l2p_proto_index(svn_fs_t *fs, svn_fs_x__txn_id_t txn_id, @@ -543,6 +535,7 @@ svn_fs_x__write_min_unpacked_rev(svn_fs_t *fs, svn_revnum_t revnum, apr_pool_t *scratch_pool) { + svn_fs_x__data_t *ffd = fs->fsap_data; const char *final_path; char buf[SVN_INT64_BUFFER_SIZE]; apr_size_t len = svn__i64toa(buf, revnum); @@ -550,8 +543,9 @@ svn_fs_x__write_min_unpacked_rev(svn_fs_t *fs, final_path = svn_fs_x__path_min_unpacked_rev(fs, scratch_pool); - SVN_ERR(svn_io_write_atomic(final_path, buf, len + 1, - final_path /* copy_perms */, scratch_pool)); + SVN_ERR(svn_io_write_atomic2(final_path, buf, len + 1, + final_path /* copy_perms */, + ffd->flush_to_disk, scratch_pool)); return SVN_NO_ERROR; } @@ -574,7 +568,7 @@ svn_fs_x__read_current(svn_revnum_t *rev, return SVN_NO_ERROR; } -/* Atomically update the 'current' file to hold the specifed REV. +/* Atomically update the 'current' file to hold the specified REV. Perform temporary allocations in SCRATCH_POOL. */ svn_error_t * svn_fs_x__write_current(svn_fs_t *fs, @@ -583,17 +577,28 @@ svn_fs_x__write_current(svn_fs_t *fs, { char *buf; const char *tmp_name, *name; + apr_file_t *file; /* Now we can just write out this line. */ buf = apr_psprintf(scratch_pool, "%ld\n", rev); name = svn_fs_x__path_current(fs, scratch_pool); - SVN_ERR(svn_io_write_unique(&tmp_name, - svn_dirent_dirname(name, scratch_pool), - buf, strlen(buf), - svn_io_file_del_none, scratch_pool)); + tmp_name = svn_fs_x__path_next(fs, scratch_pool); - return svn_fs_x__move_into_place(tmp_name, name, name, scratch_pool); + SVN_ERR(svn_io_file_open(&file, tmp_name, + APR_WRITE | APR_CREATE | APR_BUFFERED, + APR_OS_DEFAULT, scratch_pool)); + SVN_ERR(svn_io_file_write_full(file, buf, strlen(buf), NULL, + scratch_pool)); + SVN_ERR(svn_io_file_close(file, scratch_pool)); + + /* Copying permissions is a no-op on WIN32. */ + SVN_ERR(svn_io_copy_perms(name, tmp_name, scratch_pool)); + + /* Move the file into place. */ + SVN_ERR(svn_io_file_rename2(tmp_name, name, TRUE, scratch_pool)); + + return SVN_NO_ERROR; } @@ -640,22 +645,6 @@ svn_fs_x__try_stringbuf_from_file(svn_stringbuf_t **content, /* Fetch the current offset of FILE into *OFFSET_P. */ svn_error_t * -svn_fs_x__get_file_offset(apr_off_t *offset_p, - apr_file_t *file, - apr_pool_t *scratch_pool) -{ - apr_off_t offset; - - /* Note that, for buffered files, one (possibly surprising) side-effect - of this call is to flush any unwritten data to disk. */ - offset = 0; - SVN_ERR(svn_io_file_seek(file, APR_CUR, &offset, scratch_pool)); - *offset_p = offset; - - return SVN_NO_ERROR; -} - -svn_error_t * svn_fs_x__read_content(svn_stringbuf_t **content, const char *fname, apr_pool_t *result_pool) @@ -711,66 +700,33 @@ svn_fs_x__read_number_from_stream(apr_int64_t *result, return SVN_NO_ERROR; } - -/* Move a file into place from OLD_FILENAME in the transactions - directory to its final location NEW_FILENAME in the repository. On - Unix, match the permissions of the new file to the permissions of - PERMS_REFERENCE. Temporary allocations are from SCRATCH_POOL. - - This function almost duplicates svn_io_file_move(), but it tries to - guarantee a flush. */ svn_error_t * svn_fs_x__move_into_place(const char *old_filename, const char *new_filename, const char *perms_reference, + svn_fs_x__batch_fsync_t *batch, apr_pool_t *scratch_pool) { - svn_error_t *err; - + /* Copying permissions is a no-op on WIN32. */ SVN_ERR(svn_io_copy_perms(perms_reference, old_filename, scratch_pool)); + /* We use specific 'fsyncing move' Win32 API calls on Windows while the + * directory update fsync is POSIX-only. Moreover, there tend to be only + * a few moved files (1 or 2) per batch. + * + * Therefore, we use the platform-optimized "immediate" fsyncs on all + * non-POSIX platforms and the "scheduled" fsyncs on POSIX only. + */ +#if defined(SVN_ON_POSIX) /* Move the file into place. */ - err = svn_io_file_rename(old_filename, new_filename, scratch_pool); - if (err && APR_STATUS_IS_EXDEV(err->apr_err)) - { - apr_file_t *file; - - /* Can't rename across devices; fall back to copying. */ - svn_error_clear(err); - err = SVN_NO_ERROR; - SVN_ERR(svn_io_copy_file(old_filename, new_filename, TRUE, - scratch_pool)); - - /* Flush the target of the copy to disk. */ - SVN_ERR(svn_io_file_open(&file, new_filename, APR_READ, - APR_OS_DEFAULT, scratch_pool)); - /* ### BH: Does this really guarantee a flush of the data written - ### via a completely different handle on all operating systems? - ### - ### Maybe we should perform the copy ourselves instead of making - ### apr do that and flush the real handle? */ - SVN_ERR(svn_io_file_flush_to_disk(file, scratch_pool)); - SVN_ERR(svn_io_file_close(file, scratch_pool)); - } - if (err) - return svn_error_trace(err); - -#ifdef __linux__ - { - /* Linux has the unusual feature that fsync() on a file is not - enough to ensure that a file's directory entries have been - flushed to disk; you have to fsync the directory as well. - On other operating systems, we'd only be asking for trouble - by trying to open and fsync a directory. */ - const char *dirname; - apr_file_t *file; - - dirname = svn_dirent_dirname(new_filename, scratch_pool); - SVN_ERR(svn_io_file_open(&file, dirname, APR_READ, APR_OS_DEFAULT, - scratch_pool)); - SVN_ERR(svn_io_file_flush_to_disk(file, scratch_pool)); - SVN_ERR(svn_io_file_close(file, scratch_pool)); - } + SVN_ERR(svn_io_file_rename2(old_filename, new_filename, FALSE, + scratch_pool)); + + /* Schedule for synchronization. */ + SVN_ERR(svn_fs_x__batch_fsync_new_path(batch, new_filename, scratch_pool)); +#else + SVN_ERR(svn_io_file_rename2(old_filename, new_filename, TRUE, + scratch_pool)); #endif return SVN_NO_ERROR; diff --git a/subversion/libsvn_fs_x/util.h b/subversion/libsvn_fs_x/util.h index 0010723ed4bf..9622328ae771 100644 --- a/subversion/libsvn_fs_x/util.h +++ b/subversion/libsvn_fs_x/util.h @@ -20,11 +20,12 @@ * ==================================================================== */ -#ifndef SVN_LIBSVN_FS__UTIL_H -#define SVN_LIBSVN_FS__UTIL_H +#ifndef SVN_LIBSVN_FS_X_UTIL_H +#define SVN_LIBSVN_FS_X_UTIL_H #include "svn_fs.h" #include "id.h" +#include "batch_fsync.h" /* Functions for dealing with recoverable errors on mutable files * @@ -102,6 +103,12 @@ const char * svn_fs_x__path_current(svn_fs_t *fs, apr_pool_t *result_pool); +/* Return the path to the 'next' file in FS. + Perform allocation in RESULT_POOL. */ +const char * +svn_fs_x__path_next(svn_fs_t *fs, + apr_pool_t *result_pool); + /* Return the full path of the "uuid" file in FS. * The result will be allocated in RESULT_POOL. */ @@ -158,9 +165,9 @@ svn_fs_x__path_rev_packed(svn_fs_t *fs, * revision REV in FS. Allocate the result in RESULT_POOL. */ const char * -svn_fs_x__path_rev_shard(svn_fs_t *fs, - svn_revnum_t rev, - apr_pool_t *result_pool); +svn_fs_x__path_shard(svn_fs_t *fs, + svn_revnum_t rev, + apr_pool_t *result_pool); /* Return the full path of the non-packed rev file containing revision REV * in FS. Allocate the result in RESULT_POOL. @@ -183,23 +190,13 @@ svn_fs_x__path_rev_absolute(svn_fs_t *fs, svn_revnum_t rev, apr_pool_t *result_pool); -/* Return the full path of the revision properties shard directory that - * will contain the properties of revision REV in FS. - * Allocate the result in RESULT_POOL. - */ -const char * -svn_fs_x__path_revprops_shard(svn_fs_t *fs, - svn_revnum_t rev, - apr_pool_t *result_pool); - -/* Return the full path of the revision properties pack shard directory - * that will contain the packed properties of revision REV in FS. - * Allocate the result in RESULT_POOL. +/* Return the full path of the pack shard directory that will contain the + * packed revision REV in FS. Allocate the result in RESULT_POOL. */ const char * -svn_fs_x__path_revprops_pack_shard(svn_fs_t *fs, - svn_revnum_t rev, - apr_pool_t *result_pool); +svn_fs_x__path_pack_shard(svn_fs_t *fs, + svn_revnum_t rev, + apr_pool_t *result_pool); /* Return the full path of the non-packed revision properties file that * contains the props for revision REV in FS. @@ -287,15 +284,6 @@ svn_fs_x__path_txn_props(svn_fs_t *fs, svn_fs_x__txn_id_t txn_id, apr_pool_t *result_pool); -/* Return the path of the file containing the "final" transaction - * properties for the transaction identified by TXN_ID in FS. - * The result will be allocated in RESULT_POOL. - */ -const char * -svn_fs_x__path_txn_props_final(svn_fs_t *fs, - svn_fs_x__txn_id_t txn_id, - apr_pool_t *result_pool); - /* Return the path of the file containing the node and copy ID counters for * the transaction identified by TXN_ID in FS. * The result will be allocated in RESULT_POOL. @@ -381,7 +369,7 @@ svn_fs_x__check_file_buffer_numeric(const char *buf, apr_pool_t *scratch_pool); /* Set *MIN_UNPACKED_REV to the integer value read from the file returned - * by #svn_fs_fs__path_min_unpacked_rev() for FS. + * by #svn_fs_x__path_min_unpacked_rev() for FS. * Use SCRATCH_POOL for temporary allocations. */ svn_error_t * @@ -434,13 +422,6 @@ svn_fs_x__try_stringbuf_from_file(svn_stringbuf_t **content, svn_boolean_t last_attempt, apr_pool_t *result_pool); -/* Fetch the current offset of FILE into *OFFSET_P. - * Perform temporary allocations in SCRATCH_POOL. */ -svn_error_t * -svn_fs_x__get_file_offset(apr_off_t *offset_p, - apr_file_t *file, - apr_pool_t *scratch_pool); - /* Read the file FNAME and store the contents in *BUF. Allocations are performed in RESULT_POOL. */ svn_error_t * @@ -460,17 +441,20 @@ svn_fs_x__read_number_from_stream(apr_int64_t *result, svn_stream_t *stream, apr_pool_t *scratch_pool); -/* Move a file into place from OLD_FILENAME in the transactions - directory to its final location NEW_FILENAME in the repository. On - Unix, match the permissions of the new file to the permissions of - PERMS_REFERENCE. Temporary allocations are from SCRATCH_POOL. +/* Move a file into place from temporary OLD_FILENAME to its final + location NEW_FILENAME, which must be on to the same volume. Schedule + any necessary fsync calls in BATCH. On Unix, match the permissions + of the new file to the permissions of PERMS_REFERENCE. + + Temporary allocations are from SCRATCH_POOL. This function almost duplicates svn_io_file_move(), but it tries to - guarantee a flush. */ + guarantee a flush if BATCH->FLUSH_TO_DISK is non-zero. */ svn_error_t * svn_fs_x__move_into_place(const char *old_filename, const char *new_filename, const char *perms_reference, + svn_fs_x__batch_fsync_t *batch, apr_pool_t *scratch_pool); #endif diff --git a/subversion/libsvn_fs_x/verify.c b/subversion/libsvn_fs_x/verify.c index 4ea0728f7d5f..6f03201b2b8e 100644 --- a/subversion/libsvn_fs_x/verify.c +++ b/subversion/libsvn_fs_x/verify.c @@ -27,6 +27,7 @@ #include "cached_data.h" #include "rep-cache.h" +#include "revprops.h" #include "util.h" #include "index.h" @@ -141,30 +142,27 @@ verify_rep_cache(svn_fs_t *fs, * indedx NAME in the error message. Supports cancellation with CANCEL_FUNC * and CANCEL_BATON. SCRATCH_POOL is for temporary allocations. */ static svn_error_t * -verify_index_checksum(apr_file_t *file, +verify_index_checksum(svn_fs_x__revision_file_t *file, const char *name, - apr_off_t start, - apr_off_t end, - svn_checksum_t *expected, + svn_fs_x__index_info_t *index_info, svn_cancel_func_t cancel_func, void *cancel_baton, apr_pool_t *scratch_pool) { unsigned char buffer[SVN__STREAM_CHUNK_SIZE]; - apr_off_t size = end - start; + apr_off_t size = index_info->end - index_info->start; svn_checksum_t *actual; svn_checksum_ctx_t *context = svn_checksum_ctx_create(svn_checksum_md5, scratch_pool); /* Calculate the index checksum. */ - SVN_ERR(svn_io_file_seek(file, APR_SET, &start, scratch_pool)); + SVN_ERR(svn_fs_x__rev_file_seek(file, NULL, index_info->start)); while (size > 0) { apr_size_t to_read = size > sizeof(buffer) ? sizeof(buffer) : (apr_size_t)size; - SVN_ERR(svn_io_file_read_full2(file, buffer, to_read, NULL, NULL, - scratch_pool)); + SVN_ERR(svn_fs_x__rev_file_read(file, buffer, to_read)); SVN_ERR(svn_checksum_update(context, buffer, to_read)); size -= to_read; @@ -175,12 +173,13 @@ verify_index_checksum(apr_file_t *file, SVN_ERR(svn_checksum_final(&actual, context, scratch_pool)); /* Verify that it matches the expected checksum. */ - if (!svn_checksum_match(expected, actual)) + if (!svn_checksum_match(index_info->checksum, actual)) { const char *file_name; - SVN_ERR(svn_io_file_name_get(&file_name, file, scratch_pool)); - SVN_ERR(svn_checksum_mismatch_err(expected, actual, scratch_pool, + SVN_ERR(svn_fs_x__rev_file_name(&file_name, file, scratch_pool)); + SVN_ERR(svn_checksum_mismatch_err(index_info->checksum, actual, + scratch_pool, _("%s checksum mismatch in file %s"), name, file_name)); } @@ -201,20 +200,18 @@ verify_index_checksums(svn_fs_t *fs, apr_pool_t *scratch_pool) { svn_fs_x__revision_file_t *rev_file; + svn_fs_x__index_info_t l2p_index_info; + svn_fs_x__index_info_t p2l_index_info; /* Open the rev / pack file and read the footer */ - SVN_ERR(svn_fs_x__open_pack_or_rev_file(&rev_file, fs, start, - scratch_pool, scratch_pool)); - SVN_ERR(svn_fs_x__auto_read_footer(rev_file)); + SVN_ERR(svn_fs_x__rev_file_init(&rev_file, fs, start, scratch_pool)); + SVN_ERR(svn_fs_x__rev_file_l2p_info(&l2p_index_info, rev_file)); + SVN_ERR(svn_fs_x__rev_file_p2l_info(&p2l_index_info, rev_file)); /* Verify the index contents against the checksum from the footer. */ - SVN_ERR(verify_index_checksum(rev_file->file, "L2P index", - rev_file->l2p_offset, rev_file->p2l_offset, - rev_file->l2p_checksum, + SVN_ERR(verify_index_checksum(rev_file, "L2P index", &l2p_index_info, cancel_func, cancel_baton, scratch_pool)); - SVN_ERR(verify_index_checksum(rev_file->file, "P2L index", - rev_file->p2l_offset, rev_file->footer_offset, - rev_file->p2l_checksum, + SVN_ERR(verify_index_checksum(rev_file, "P2L index", &p2l_index_info, cancel_func, cancel_baton, scratch_pool)); /* Done. */ @@ -242,8 +239,7 @@ compare_l2p_to_p2l_index(svn_fs_t *fs, /* common file access structure */ svn_fs_x__revision_file_t *rev_file; - SVN_ERR(svn_fs_x__open_pack_or_rev_file(&rev_file, fs, start, scratch_pool, - iterpool)); + SVN_ERR(svn_fs_x__rev_file_init(&rev_file, fs, start, scratch_pool)); /* determine the range of items to check for each revision */ SVN_ERR(svn_fs_x__l2p_get_max_ids(&max_ids, fs, start, count, scratch_pool, @@ -334,8 +330,7 @@ compare_p2l_to_l2p_index(svn_fs_t *fs, /* common file access structure */ svn_fs_x__revision_file_t *rev_file; - SVN_ERR(svn_fs_x__open_pack_or_rev_file(&rev_file, fs, start, scratch_pool, - iterpool)); + SVN_ERR(svn_fs_x__rev_file_init(&rev_file, fs, start, scratch_pool)); /* get the size of the rev / pack file as covered by the P2L index */ SVN_ERR(svn_fs_x__p2l_get_max_offset(&max_offset, fs, rev_file, start, @@ -424,7 +419,7 @@ compare_p2l_to_l2p_index(svn_fs_t *fs, * exceed STREAM_THRESHOLD. Use SCRATCH_POOL for temporary allocations. */ static svn_error_t * -expect_buffer_nul(apr_file_t *file, +expect_buffer_nul(svn_fs_x__revision_file_t *file, apr_off_t size, apr_pool_t *scratch_pool) { @@ -439,8 +434,7 @@ expect_buffer_nul(apr_file_t *file, /* read the whole data block; error out on failure */ data.chunks[(size - 1)/ sizeof(apr_uint64_t)] = 0; - SVN_ERR(svn_io_file_read_full2(file, data.buffer, size, NULL, NULL, - scratch_pool)); + SVN_ERR(svn_fs_x__rev_file_read(file, data.buffer, size)); /* chunky check */ for (i = 0; i < size / sizeof(apr_uint64_t); ++i) @@ -454,8 +448,8 @@ expect_buffer_nul(apr_file_t *file, const char *file_name; apr_off_t offset; - SVN_ERR(svn_io_file_name_get(&file_name, file, scratch_pool)); - SVN_ERR(svn_fs_x__get_file_offset(&offset, file, scratch_pool)); + SVN_ERR(svn_fs_x__rev_file_name(&file_name, file, scratch_pool)); + SVN_ERR(svn_fs_x__rev_file_offset(&offset, file)); offset -= size - i; return svn_error_createf(SVN_ERR_FS_CORRUPT, NULL, @@ -472,7 +466,7 @@ expect_buffer_nul(apr_file_t *file, * Use SCRATCH_POOL for temporary allocations. */ static svn_error_t * -read_all_nul(apr_file_t *file, +read_all_nul(svn_fs_x__revision_file_t *file, apr_off_t size, apr_pool_t *scratch_pool) { @@ -490,7 +484,7 @@ read_all_nul(apr_file_t *file, * in error message. Allocate temporary data in SCRATCH_POOL. */ static svn_error_t * -expected_checksum(apr_file_t *file, +expected_checksum(svn_fs_x__revision_file_t *file, svn_fs_x__p2l_entry_t *entry, apr_uint32_t actual, apr_pool_t *scratch_pool) @@ -499,8 +493,7 @@ expected_checksum(apr_file_t *file, { const char *file_name; - SVN_ERR(svn_io_file_name_get(&file_name, file, scratch_pool)); - SVN_ERR(svn_io_file_name_get(&file_name, file, scratch_pool)); + SVN_ERR(svn_fs_x__rev_file_name(&file_name, file, scratch_pool)); return svn_error_createf(SVN_ERR_FS_CORRUPT, NULL, _("Checksum mismatch in item at offset %s of " "length %s bytes in file %s"), @@ -517,15 +510,14 @@ expected_checksum(apr_file_t *file, * exceed STREAM_THRESHOLD. Use SCRATCH_POOL for temporary allocations. */ static svn_error_t * -expected_buffered_checksum(apr_file_t *file, +expected_buffered_checksum(svn_fs_x__revision_file_t *file, svn_fs_x__p2l_entry_t *entry, apr_pool_t *scratch_pool) { unsigned char buffer[STREAM_THRESHOLD]; SVN_ERR_ASSERT(entry->size <= STREAM_THRESHOLD); - SVN_ERR(svn_io_file_read_full2(file, buffer, (apr_size_t)entry->size, - NULL, NULL, scratch_pool)); + SVN_ERR(svn_fs_x__rev_file_read(file, buffer, (apr_size_t)entry->size)); SVN_ERR(expected_checksum(file, entry, svn__fnv1a_32x4(buffer, (apr_size_t)entry->size), scratch_pool)); @@ -538,7 +530,7 @@ expected_buffered_checksum(apr_file_t *file, * Use SCRATCH_POOL for temporary allocations. */ static svn_error_t * -expected_streamed_checksum(apr_file_t *file, +expected_streamed_checksum(svn_fs_x__revision_file_t *file, svn_fs_x__p2l_entry_t *entry, apr_pool_t *scratch_pool) { @@ -553,8 +545,7 @@ expected_streamed_checksum(apr_file_t *file, apr_size_t to_read = size > sizeof(buffer) ? sizeof(buffer) : (apr_size_t)size; - SVN_ERR(svn_io_file_read_full2(file, buffer, to_read, NULL, NULL, - scratch_pool)); + SVN_ERR(svn_fs_x__rev_file_read(file, buffer, to_read)); SVN_ERR(svn_checksum_update(context, buffer, to_read)); size -= to_read; } @@ -588,28 +579,27 @@ compare_p2l_to_rev(svn_fs_t *fs, apr_off_t max_offset; apr_off_t offset = 0; svn_fs_x__revision_file_t *rev_file; + svn_fs_x__index_info_t l2p_index_info; /* open the pack / rev file that is covered by the p2l index */ - SVN_ERR(svn_fs_x__open_pack_or_rev_file(&rev_file, fs, start, scratch_pool, - iterpool)); + SVN_ERR(svn_fs_x__rev_file_init(&rev_file, fs, start, scratch_pool)); /* check file size vs. range covered by index */ - SVN_ERR(svn_fs_x__auto_read_footer(rev_file)); + SVN_ERR(svn_fs_x__rev_file_l2p_info(&l2p_index_info, rev_file)); SVN_ERR(svn_fs_x__p2l_get_max_offset(&max_offset, fs, rev_file, start, scratch_pool)); - if (rev_file->l2p_offset != max_offset) + if (l2p_index_info.start != max_offset) return svn_error_createf(SVN_ERR_FS_INDEX_INCONSISTENT, NULL, _("File size of %s for revision r%ld does " "not match p2l index size of %s"), apr_off_t_toa(scratch_pool, - rev_file->l2p_offset), + l2p_index_info.start), start, apr_off_t_toa(scratch_pool, max_offset)); - SVN_ERR(svn_io_file_aligned_seek(rev_file->file, ffd->block_size, NULL, 0, - scratch_pool)); + SVN_ERR(svn_fs_x__rev_file_seek(rev_file, NULL, 0)); /* for all offsets in the file, get the P2L index entries and check them against the L2P index */ @@ -627,8 +617,7 @@ compare_p2l_to_rev(svn_fs_t *fs, /* The above might have moved the file pointer. * Ensure we actually start reading at OFFSET. */ - SVN_ERR(svn_io_file_aligned_seek(rev_file->file, ffd->block_size, - NULL, offset, iterpool)); + SVN_ERR(svn_fs_x__rev_file_seek(rev_file, NULL, offset)); /* process all entries (and later continue with the next block) */ for (i = 0; i < entries->nelts; ++i) @@ -661,15 +650,15 @@ compare_p2l_to_rev(svn_fs_t *fs, { /* skip filler entry at the end of the p2l index */ if (entry->offset != max_offset) - SVN_ERR(read_all_nul(rev_file->file, entry->size, iterpool)); + SVN_ERR(read_all_nul(rev_file, entry->size, iterpool)); } else { if (entry->size < STREAM_THRESHOLD) - SVN_ERR(expected_buffered_checksum(rev_file->file, entry, + SVN_ERR(expected_buffered_checksum(rev_file, entry, iterpool)); else - SVN_ERR(expected_streamed_checksum(rev_file->file, entry, + SVN_ERR(expected_streamed_checksum(rev_file, entry, iterpool)); } @@ -703,6 +692,10 @@ verify_revprops(svn_fs_t *fs, svn_revnum_t revision; apr_pool_t *iterpool = svn_pool_create(scratch_pool); + /* Invalidate the revprop generation once. + * Use the cache inside the loop to speed up packed revprop access. */ + svn_fs_x__invalidate_revprop_generation(fs); + for (revision = start; revision < end; ++revision) { svn_string_t *date; @@ -713,7 +706,7 @@ verify_revprops(svn_fs_t *fs, /* Access the svn:date revprop. * This implies parsing all revprops for that revision. */ SVN_ERR(svn_fs_x__revision_prop(&date, fs, revision, - SVN_PROP_REVISION_DATE, + SVN_PROP_REVISION_DATE, FALSE, iterpool, iterpool)); /* The time stamp is the only revprop that, if given, needs to @@ -790,8 +783,15 @@ verify_metadata_consistency(svn_fs_t *fs, /* concurrent packing is one of the reasons why verification may fail. Make sure, we operate on up-to-date information. */ if (err) - SVN_ERR(svn_fs_x__read_min_unpacked_rev(&ffd->min_unpacked_rev, - fs, scratch_pool)); + { + svn_error_t *err2 + = svn_fs_x__read_min_unpacked_rev(&ffd->min_unpacked_rev, + fs, scratch_pool); + + /* Be careful to not leak ERR. */ + if (err2) + return svn_error_trace(svn_error_compose_create(err, err2)); + } /* retry the whole shard if it got packed in the meantime */ if (err && count != svn_fs_x__pack_size(fs, revision)) @@ -824,14 +824,14 @@ svn_fs_x__verify(svn_fs_t *fs, void *cancel_baton, apr_pool_t *scratch_pool) { - svn_fs_x__data_t *ffd = fs->fsap_data; - svn_revnum_t youngest = ffd->youngest_rev_cache; /* cache is current */ - /* Input validation. */ if (! SVN_IS_VALID_REVNUM(start)) start = 0; if (! SVN_IS_VALID_REVNUM(end)) - end = youngest; + { + SVN_ERR(svn_fs_x__youngest_rev(&end, fs, scratch_pool)); + } + SVN_ERR(svn_fs_x__ensure_revision_exists(start, fs, scratch_pool)); SVN_ERR(svn_fs_x__ensure_revision_exists(end, fs, scratch_pool)); diff --git a/subversion/libsvn_fs_x/verify.h b/subversion/libsvn_fs_x/verify.h index 805f65496150..8b76be9ea0b9 100644 --- a/subversion/libsvn_fs_x/verify.h +++ b/subversion/libsvn_fs_x/verify.h @@ -1,4 +1,4 @@ -/* verify.h : verification interface of the native filesystem layer +/* verify.h : verification interface of the FSX filesystem * * ==================================================================== * Licensed to the Apache Software Foundation (ASF) under one @@ -20,8 +20,8 @@ * ==================================================================== */ -#ifndef SVN_LIBSVN_FS__VERIFY_H -#define SVN_LIBSVN_FS__VERIFY_H +#ifndef SVN_LIBSVN_FS_X_VERIFY_H +#define SVN_LIBSVN_FS_X_VERIFY_H #include "fs.h" |