You are viewing a plain text version of this content. The canonical link for it is here.
Posted to cvs@httpd.apache.org by ji...@apache.org on 2014/05/01 13:43:45 UTC
svn commit: r1591622 [18/33] - in /httpd/mod_spdy/trunk: ./ base/
base/base.xcodeproj/ base/metrics/ build/ build/all.xcodeproj/
build/build_util.xcodeproj/ build/install.xcodeproj/ build/internal/
build/linux/ build/mac/ build/util/ build/win/ install...
Added: httpd/mod_spdy/trunk/net/instaweb/util/file_cache.cc
URL: http://svn.apache.org/viewvc/httpd/mod_spdy/trunk/net/instaweb/util/file_cache.cc?rev=1591622&view=auto
==============================================================================
--- httpd/mod_spdy/trunk/net/instaweb/util/file_cache.cc (added)
+++ httpd/mod_spdy/trunk/net/instaweb/util/file_cache.cc Thu May 1 11:43:36 2014
@@ -0,0 +1,344 @@
+/*
+ * Copyright 2010 Google Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+// Author: lsong@google.com (Libo Song)
+
+#include "net/instaweb/util/public/file_cache.h"
+
+#include <cstddef>
+#include <cstdlib>
+#include <vector>
+#include <queue>
+#include "base/scoped_ptr.h"
+#include "net/instaweb/util/public/basictypes.h"
+#include "net/instaweb/util/public/cache_interface.h"
+#include "net/instaweb/util/public/file_system.h"
+#include "net/instaweb/util/public/filename_encoder.h"
+#include "net/instaweb/util/public/function.h"
+#include "net/instaweb/util/public/hasher.h"
+#include "net/instaweb/util/public/message_handler.h"
+#include "net/instaweb/util/public/null_message_handler.h"
+#include "net/instaweb/util/public/shared_string.h"
+#include "net/instaweb/util/public/slow_worker.h"
+#include "net/instaweb/util/public/string.h"
+#include "net/instaweb/util/public/string_util.h"
+#include "net/instaweb/util/public/timer.h"
+
+namespace net_instaweb {
+
+namespace { // For structs used only in Clean().
+
+class CacheFileInfo {
+ public:
+ CacheFileInfo(int64 size, int64 atime, const GoogleString& name)
+ : size_(size), atime_(atime), name_(name) {}
+ int64 size_;
+ int64 atime_;
+ GoogleString name_;
+ private:
+ DISALLOW_COPY_AND_ASSIGN(CacheFileInfo);
+};
+
+struct CompareByAtime {
+ public:
+ bool operator()(const CacheFileInfo* one,
+ const CacheFileInfo* two) const {
+ return one->atime_ < two->atime_;
+ }
+};
+
+} // namespace for structs used only in Clean().
+
+class FileCache::CacheCleanFunction : public Function {
+ public:
+ CacheCleanFunction(FileCache* cache, int64 next_clean_time_ms)
+ : cache_(cache),
+ next_clean_time_ms_(next_clean_time_ms) {}
+ virtual ~CacheCleanFunction() {}
+ virtual void Run() {
+ cache_->last_conditional_clean_result_ =
+ cache_->CleanWithLocking(next_clean_time_ms_);
+ }
+
+ private:
+ FileCache* cache_;
+ int64 next_clean_time_ms_;
+ DISALLOW_COPY_AND_ASSIGN(CacheCleanFunction);
+};
+
+// Filenames for the next scheduled clean time and the lockfile. In
+// order to prevent these from colliding with actual cachefiles, they
+// contain characters that our filename encoder would escape.
+const char FileCache::kCleanTimeName[] = "!clean!time!";
+const char FileCache::kCleanLockName[] = "!clean!lock!";
+
+// TODO(abliss): remove policy from constructor; provide defaults here
+// and setters below.
+FileCache::FileCache(const GoogleString& path, FileSystem* file_system,
+ SlowWorker* worker,
+ FilenameEncoder* filename_encoder,
+ CachePolicy* policy,
+ MessageHandler* handler)
+ : path_(path),
+ file_system_(file_system),
+ worker_(worker),
+ filename_encoder_(filename_encoder),
+ message_handler_(handler),
+ cache_policy_(policy),
+ path_length_limit_(file_system_->MaxPathLength(path)),
+ clean_time_path_(path) {
+ // NOTE(abliss): We don't want all the caches racing for the
+ // lock at startup, so each one gets a random offset.
+ next_clean_ms_ = policy->timer->NowMs()
+ + (random() % policy->clean_interval_ms);
+ EnsureEndsInSlash(&clean_time_path_);
+ clean_time_path_ += kCleanTimeName;
+}
+
+FileCache::~FileCache() {
+}
+
+void FileCache::Get(const GoogleString& key, Callback* callback) {
+ GoogleString filename;
+ bool ret = EncodeFilename(key, &filename);
+ if (ret) {
+ GoogleString* buffer = callback->value()->get();
+
+ // Suppress read errors. Note that we want to show Write errors,
+ // as they likely indicate a permissions or disk-space problem
+ // which is best not eaten. It's cheap enough to construct
+ // a NullMessageHandler on the stack when we want one.
+ NullMessageHandler null_handler;
+ ret = file_system_->ReadFile(filename.c_str(), buffer, &null_handler);
+ }
+ callback->Done(ret ? kAvailable : kNotFound);
+}
+
+void FileCache::Put(const GoogleString& key, SharedString* value) {
+ GoogleString filename;
+ if (EncodeFilename(key, &filename)) {
+ const GoogleString& buffer = **value;
+ GoogleString temp_filename;
+ if (file_system_->WriteTempFile(filename, buffer,
+ &temp_filename, message_handler_)) {
+ file_system_->RenameFile(temp_filename.c_str(), filename.c_str(),
+ message_handler_);
+ }
+ }
+ CleanIfNeeded();
+}
+
+void FileCache::Delete(const GoogleString& key) {
+ GoogleString filename;
+ if (!EncodeFilename(key, &filename)) {
+ return;
+ }
+ file_system_->RemoveFile(filename.c_str(), message_handler_);
+ return;
+}
+
+bool FileCache::EncodeFilename(const GoogleString& key,
+ GoogleString* filename) {
+ GoogleString prefix = path_;
+ // TODO(abliss): unify and make explicit everyone's assumptions
+ // about trailing slashes.
+ EnsureEndsInSlash(&prefix);
+ filename_encoder_->Encode(prefix, key, filename);
+
+ // Make sure the length isn't too big for filesystem to handle; if it is
+ // just name the object using a hash.
+ if (static_cast<int>(filename->length()) > path_length_limit_) {
+ filename_encoder_->Encode(prefix, cache_policy_->hasher->Hash(key),
+ filename);
+ }
+
+ return true;
+}
+
+bool FileCache::Clean(int64 target_size) {
+ StringVector files;
+ int64 file_size;
+ int64 file_atime;
+ int64 total_size = 0;
+
+ message_handler_->Message(kInfo,
+ "Checking cache size against target %ld",
+ static_cast<long>(target_size));
+
+ if (!file_system_->RecursiveDirSize(path_, &total_size, message_handler_)) {
+ return false;
+ }
+
+ // TODO(jmarantz): gcc 4.1 warns about double/int64 comparisons here,
+ // but this really should be factored into a settable member var.
+ if (total_size < ((target_size * 5) / 4)) {
+ message_handler_->Message(kInfo,
+ "File cache size is %ld; no cleanup needed.",
+ static_cast<long>(total_size));
+ return true;
+ }
+ message_handler_->Message(kInfo,
+ "File cache size is %ld; beginning cleanup.",
+ static_cast<long>(total_size));
+ bool everything_ok = true;
+ everything_ok &= file_system_->ListContents(path_, &files, message_handler_);
+
+ // We will now iterate over the entire directory and its children,
+ // keeping a heap of files to be deleted. Our goal is to delete the
+ // oldest set of files that sum to enough space to bring us below
+ // our target.
+ std::priority_queue<CacheFileInfo*, std::vector<CacheFileInfo*>,
+ CompareByAtime> heap;
+ int64 total_heap_size = 0;
+ // TODO(jmarantz): gcc 4.1 warns about double/int64 comparisons here,
+ // but this really should be factored into a settable member var.
+ int64 target_heap_size = total_size - ((target_size * 3 / 4));
+
+ GoogleString prefix = path_;
+ EnsureEndsInSlash(&prefix);
+ for (size_t i = 0; i < files.size(); i++) {
+ GoogleString file_name = files[i];
+ BoolOrError isDir = file_system_->IsDir(file_name.c_str(),
+ message_handler_);
+ if (isDir.is_error()) {
+ return false;
+ } else if (clean_time_path_.compare(file_name) == 0) {
+ // Don't clean the clean_time file! It ought to be the newest file (and
+ // very small) so the following algorithm would normally not delete it
+ // anyway. But on some systems (e.g. mounted noatime?) it was getting
+ // deleted.
+ continue;
+ } else if (isDir.is_true()) {
+ // add files in this directory to the end of the vector, to be
+ // examined later.
+ everything_ok &= file_system_->ListContents(file_name, &files,
+ message_handler_);
+ } else {
+ everything_ok &= file_system_->Size(file_name, &file_size,
+ message_handler_);
+ everything_ok &= file_system_->Atime(file_name, &file_atime,
+ message_handler_);
+ // If our heap is still too small; add everything in.
+ // Otherwise, add the file in only if it's older than the newest
+ // thing in the heap.
+ if ((total_heap_size < target_heap_size) ||
+ (file_atime < heap.top()->atime_)) {
+ CacheFileInfo* info =
+ new CacheFileInfo(file_size, file_atime, file_name);
+ heap.push(info);
+ total_heap_size += file_size;
+ // Now remove new things from the heap which are not needed
+ // to keep the heap size over its target size.
+ while (total_heap_size - heap.top()->size_ > target_heap_size) {
+ total_heap_size -= heap.top()->size_;
+ delete heap.top();
+ heap.pop();
+ }
+ }
+ }
+ }
+ for (size_t i = heap.size(); i > 0; i--) {
+ everything_ok &= file_system_->RemoveFile(heap.top()->name_.c_str(),
+ message_handler_);
+ delete heap.top();
+ heap.pop();
+ }
+ message_handler_->Message(kInfo,
+ "File cache cleanup complete; freed %ld bytes\n",
+ static_cast<long>(total_heap_size));
+ return everything_ok;
+}
+
+bool FileCache::CleanWithLocking(int64 next_clean_time_ms) {
+ bool to_return = false;
+
+ GoogleString lock_name(path_);
+ EnsureEndsInSlash(&lock_name);
+ lock_name += kCleanLockName;
+ if (file_system_->TryLockWithTimeout(
+ lock_name, Timer::kHourMs, message_handler_).is_true()) {
+ // Update the timestamp file..
+ next_clean_ms_ = next_clean_time_ms;
+ file_system_->WriteFile(clean_time_path_.c_str(),
+ Integer64ToString(next_clean_time_ms),
+ message_handler_);
+
+ // Now actually clean.
+ to_return = Clean(cache_policy_->target_size);
+ file_system_->Unlock(lock_name, message_handler_);
+ }
+ return to_return;
+}
+
+bool FileCache::ShouldClean(int64* suggested_next_clean_time_ms) {
+ bool to_return = false;
+ const int64 now_ms = cache_policy_->timer->NowMs();
+ if (now_ms < next_clean_ms_) {
+ *suggested_next_clean_time_ms = next_clean_ms_; // No change yet.
+ return false;
+ }
+
+ GoogleString clean_time_str;
+ int64 clean_time_ms = 0;
+ int64 new_clean_time_ms = now_ms + cache_policy_->clean_interval_ms;
+ NullMessageHandler null_handler;
+ if (file_system_->ReadFile(clean_time_path_.c_str(), &clean_time_str,
+ &null_handler)) {
+ StringToInt64(clean_time_str, &clean_time_ms);
+ } else {
+ message_handler_->Message(
+ kWarning, "Failed to read cache clean timestamp %s. "
+ " Doing an extra cache clean to be safe.", clean_time_path_.c_str());
+ }
+
+ // If the "clean time" written in the file is older than now, we
+ // clean.
+ if (clean_time_ms < now_ms) {
+ message_handler_->Message(kInfo,
+ "Need to check cache size against target %ld",
+ static_cast<long>(cache_policy_->target_size));
+ to_return = true;
+ }
+ // If the "clean time" is later than now plus one interval, something
+ // went wrong (like the system clock moving backwards or the file
+ // getting corrupt) so we clean and reset it.
+ if (clean_time_ms > new_clean_time_ms) {
+ message_handler_->Message(kError,
+ "Next scheduled file cache clean time %s"
+ " is implausibly remote. Cleaning now.",
+ Integer64ToString(clean_time_ms).c_str());
+ to_return = true;
+ }
+
+ *suggested_next_clean_time_ms = new_clean_time_ms;
+ return to_return;
+}
+
+void FileCache::CleanIfNeeded() {
+ if (worker_ != NULL) {
+ int64 suggested_next_clean_time_ms;
+ last_conditional_clean_result_ = false;
+ if (ShouldClean(&suggested_next_clean_time_ms)) {
+ worker_->Start();
+ worker_->RunIfNotBusy(
+ new CacheCleanFunction(this, suggested_next_clean_time_ms));
+ } else {
+ next_clean_ms_ = suggested_next_clean_time_ms;
+ }
+ }
+}
+
+} // namespace net_instaweb
Added: httpd/mod_spdy/trunk/net/instaweb/util/file_cache_test.cc
URL: http://svn.apache.org/viewvc/httpd/mod_spdy/trunk/net/instaweb/util/file_cache_test.cc?rev=1591622&view=auto
==============================================================================
--- httpd/mod_spdy/trunk/net/instaweb/util/file_cache_test.cc (added)
+++ httpd/mod_spdy/trunk/net/instaweb/util/file_cache_test.cc Thu May 1 11:43:36 2014
@@ -0,0 +1,199 @@
+/*
+ * Copyright 2010 Google Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+// Author: lsong@google.com (Libo Song)
+
+// Unit-test the file cache
+#include "net/instaweb/util/public/file_cache.h"
+
+#include <unistd.h>
+
+#include "base/scoped_ptr.h"
+#include "net/instaweb/util/cache_test_base.h"
+#include "net/instaweb/util/public/basictypes.h"
+#include "net/instaweb/util/public/filename_encoder.h"
+#include "net/instaweb/util/public/google_message_handler.h"
+#include "net/instaweb/util/public/gtest.h"
+#include "net/instaweb/util/public/md5_hasher.h"
+#include "net/instaweb/util/public/mem_file_system.h"
+#include "net/instaweb/util/public/mock_timer.h"
+#include "net/instaweb/util/public/slow_worker.h"
+#include "net/instaweb/util/public/string.h"
+#include "net/instaweb/util/public/string_util.h"
+#include "net/instaweb/util/public/thread_system.h"
+#include "net/instaweb/util/public/timer.h"
+
+namespace net_instaweb {
+
+class FileCacheTest : public CacheTestBase {
+ protected:
+ FileCacheTest()
+ : thread_system_(ThreadSystem::CreateThreadSystem()),
+ worker_(thread_system_.get()),
+ mock_timer_(0),
+ file_system_(thread_system_.get(), &mock_timer_),
+ kCleanIntervalMs(Timer::kMinuteMs),
+ kTargetSize(12), // Small enough to overflow with a few strings.
+ cache_(GTestTempDir(), &file_system_, &worker_,
+ &filename_encoder_,
+ new FileCache::CachePolicy(
+ &mock_timer_, &hasher_, kCleanIntervalMs, kTargetSize),
+ &message_handler_) {
+ // TODO(jmarantz): consider using mock_thread_system if we want
+ // explicit control of time. For now, just mutex-protect the
+ // MockTimer.
+ mock_timer_.set_mutex(thread_system_->NewMutex());
+ file_system_.set_advance_time_on_update(true, &mock_timer_);
+ }
+
+ void CheckCleanTimestamp(int64 min_time_ms) {
+ GoogleString buffer;
+ file_system_.ReadFile(cache_.clean_time_path_.c_str(), &buffer,
+ &message_handler_);
+ int64 clean_time_ms;
+ StringToInt64(buffer, &clean_time_ms);
+ EXPECT_LT(min_time_ms, clean_time_ms);
+ }
+
+ virtual void SetUp() {
+ worker_.Start();
+ file_system_.Clear();
+ file_system_.set_atime_enabled(true);
+ }
+
+ virtual CacheInterface* Cache() { return &cache_; }
+ virtual void SanityCheck() { }
+
+ bool Clean(int64 size) { return cache_.Clean(size); }
+ bool CheckClean() {
+ cache_.CleanIfNeeded();
+ while (worker_.IsBusy()) {
+ usleep(10);
+ }
+ return cache_.last_conditional_clean_result_;
+ }
+
+ protected:
+ scoped_ptr<ThreadSystem> thread_system_;
+ MD5Hasher hasher_;
+ SlowWorker worker_;
+ MockTimer mock_timer_;
+ MemFileSystem file_system_;
+ FilenameEncoder filename_encoder_;
+ const int64 kCleanIntervalMs;
+ const int64 kTargetSize;
+ FileCache cache_;
+ GoogleMessageHandler message_handler_;
+
+ private:
+ DISALLOW_COPY_AND_ASSIGN(FileCacheTest);
+};
+
+// Simple flow of putting in an item, getting it, deleting it.
+TEST_F(FileCacheTest, PutGetDelete) {
+ CheckPut("Name", "Value");
+ CheckGet("Name", "Value");
+ CheckNotFound("Another Name");
+
+ CheckPut("Name", "NewValue");
+ CheckGet("Name", "NewValue");
+
+ cache_.Delete("Name");
+ CheckNotFound("Name");
+}
+
+// Throw a bunch of files into the cache and verify that they are
+// evicted sensibly.
+TEST_F(FileCacheTest, Clean) {
+ // Make some "directory" entries so that the mem_file_system recurses
+ // correctly.
+ GoogleString dir1 = GTestTempDir() + "/a/";
+ GoogleString dir2 = GTestTempDir() + "/b/";
+ EXPECT_TRUE(file_system_.MakeDir(dir1.c_str(), &message_handler_));
+ EXPECT_TRUE(file_system_.MakeDir(dir2.c_str(), &message_handler_));
+ // Commonly-used keys
+ const char* names1[] = {"a1", "a2", "a/3"};
+ const char* values1[] = {"a2", "a234", "a2345678"};
+ // Less common keys
+ const char* names2[] =
+ {"b/1", "b2", "b3", "b4", "b5", "b6", "b7", "b8", "b9"};
+ const char* values2[] = {"b2", "b234", "b2345678",
+ "b2", "b234", "b2345678",
+ "b2", "b234", "b2345678"};
+ for (int i = 0; i < 3; i++) {
+ CheckPut(names1[i], values1[i]);
+ }
+ for (int i = 0; i < 9; i++) {
+ CheckPut(names2[i], values2[i]);
+ }
+ int64 total_size = 0;
+ EXPECT_TRUE(
+ file_system_.RecursiveDirSize(GTestTempDir(), &total_size,
+ &message_handler_));
+ EXPECT_EQ((2 + 4 + 8) * 4, total_size);
+
+ // Clean should not remove anything if target is bigger than total size.
+ EXPECT_TRUE(Clean(total_size + 1));
+ for (int i = 0; i < 27; i++) {
+ // This pattern represents more common usage of the names1 files.
+ CheckGet(names1[i % 3], values1[i % 3]);
+ CheckGet(names2[i % 9], values2[i % 9]);
+ }
+
+ // TODO(jmarantz): gcc 4.1 warns about double/int64 comparisons here,
+ // but this really should be factored into a settable member var.
+ int64 target_size = (4 * total_size) / 5 - 1;
+ EXPECT_TRUE(Clean(target_size));
+ // Common files should stay
+ for (int i = 0; i < 3; i++) {
+ CheckGet(names1[i], values1[i]);
+ }
+ // Some of the less common files should be gone
+ for (int i = 0; i < 3; i++) {
+ CheckNotFound(names2[i]);
+ }
+}
+
+// Test the auto-cleaning behavior
+TEST_F(FileCacheTest, CheckClean) {
+ CheckPut("Name1", "Value1");
+ // Cache should not clean at first.
+ EXPECT_FALSE(CheckClean());
+ mock_timer_.SleepMs(kCleanIntervalMs + 1);
+ // Because there's no timestamp, the cache should be cleaned.
+ int64 time_ms = mock_timer_.NowUs() / 1000;
+ EXPECT_TRUE(CheckClean());
+ // .. but since we're under the desired size, nothing should be removed.
+ CheckGet("Name1", "Value1");
+ // Check that the timestamp was written correctly.
+ CheckCleanTimestamp(time_ms);
+
+ // Make the cache oversize
+ CheckPut("Name2", "Value2");
+ CheckPut("Name3", "Value3");
+ // Not enough time has elapsed.
+ EXPECT_FALSE(CheckClean());
+ mock_timer_.SleepMs(kCleanIntervalMs + 1);
+ // Now we should clean. This should work even if atime doesn't work as we
+ // expect.
+ file_system_.set_atime_enabled(false);
+ time_ms = mock_timer_.NowUs() / 1000;
+ EXPECT_TRUE(CheckClean());
+ // And the timestamp should be updated.
+ CheckCleanTimestamp(time_ms);
+}
+
+} // namespace net_instaweb
Added: httpd/mod_spdy/trunk/net/instaweb/util/file_message_handler.cc
URL: http://svn.apache.org/viewvc/httpd/mod_spdy/trunk/net/instaweb/util/file_message_handler.cc?rev=1591622&view=auto
==============================================================================
--- httpd/mod_spdy/trunk/net/instaweb/util/file_message_handler.cc (added)
+++ httpd/mod_spdy/trunk/net/instaweb/util/file_message_handler.cc Thu May 1 11:43:36 2014
@@ -0,0 +1,55 @@
+/*
+ * Copyright 2010 Google Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+// Author: jmarantz@google.com (Joshua Marantz)
+
+#include "net/instaweb/util/public/file_message_handler.h"
+
+#include <cstdarg>
+#include <cstdio>
+#include <cstdlib>
+
+#include "net/instaweb/util/public/message_handler.h"
+
+namespace net_instaweb {
+
+FileMessageHandler::FileMessageHandler(FILE* file) : file_(file) {
+}
+
+void FileMessageHandler::MessageVImpl(MessageType type, const char* msg,
+ va_list args) {
+ fprintf(file_, "%s: ", MessageTypeToString(type));
+ vfprintf(file_, msg, args);
+ fputc('\n', file_);
+
+ if (type == kFatal) {
+ abort();
+ }
+}
+
+void FileMessageHandler::FileMessageVImpl(MessageType type,
+ const char* filename, int line,
+ const char *msg, va_list args) {
+ fprintf(file_, "%s: %s:%d: ", MessageTypeToString(type), filename, line);
+ vfprintf(file_, msg, args);
+ fputc('\n', file_);
+
+ if (type == kFatal) {
+ abort();
+ }
+}
+
+} // namespace net_instaweb
Added: httpd/mod_spdy/trunk/net/instaweb/util/file_system.cc
URL: http://svn.apache.org/viewvc/httpd/mod_spdy/trunk/net/instaweb/util/file_system.cc?rev=1591622&view=auto
==============================================================================
--- httpd/mod_spdy/trunk/net/instaweb/util/file_system.cc (added)
+++ httpd/mod_spdy/trunk/net/instaweb/util/file_system.cc Thu May 1 11:43:36 2014
@@ -0,0 +1,199 @@
+/*
+ * Copyright 2010 Google Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+// Author: jmarantz@google.com (Joshua Marantz)
+
+#include <cstddef>
+
+#include "net/instaweb/util/public/basictypes.h"
+#include "net/instaweb/util/public/file_system.h"
+#include "net/instaweb/util/public/message_handler.h"
+#include "net/instaweb/util/public/string.h"
+#include "net/instaweb/util/public/string_util.h"
+#include "net/instaweb/util/public/string_writer.h"
+#include "net/instaweb/util/public/writer.h"
+#include "net/instaweb/util/stack_buffer.h"
+
+namespace net_instaweb {
+
+FileSystem::~FileSystem() {
+}
+
+FileSystem::File::~File() {
+}
+
+FileSystem::InputFile::~InputFile() {
+}
+
+FileSystem::OutputFile::~OutputFile() {
+}
+
+int FileSystem::MaxPathLength(const StringPiece& base) const {
+ return 8192;
+}
+
+bool FileSystem::ReadFile(const char* filename, GoogleString* buffer,
+ MessageHandler* message_handler) {
+ StringWriter writer(buffer);
+ return ReadFile(filename, &writer, message_handler);
+}
+
+bool FileSystem::ReadFile(const char* filename, Writer* writer,
+ MessageHandler* message_handler) {
+ InputFile* input_file = OpenInputFile(filename, message_handler);
+ bool ret = false;
+ if (input_file != NULL) {
+ char buf[kStackBufferSize];
+ int nread;
+ ret = true;
+ while (ret && ((nread = input_file->Read(
+ buf, sizeof(buf), message_handler)) > 0)) {
+ ret = writer->Write(StringPiece(buf, nread), message_handler);
+ }
+ ret &= (nread == 0);
+ ret &= Close(input_file, message_handler);
+ }
+ return ret;
+}
+
+bool FileSystem::WriteFile(const char* filename, const StringPiece& buffer,
+ MessageHandler* message_handler) {
+ OutputFile* output_file = OpenOutputFile(filename, message_handler);
+ bool ret = false;
+ if (output_file != NULL) {
+ ret = output_file->Write(buffer, message_handler);
+ ret &= output_file->SetWorldReadable(message_handler);
+ ret &= Close(output_file, message_handler);
+ }
+ return ret;
+}
+
+bool FileSystem::WriteTempFile(const StringPiece& prefix_name,
+ const StringPiece& buffer,
+ GoogleString* filename,
+ MessageHandler* message_handler) {
+ OutputFile* output_file = OpenTempFile(prefix_name, message_handler);
+ bool ok = (output_file != NULL);
+ if (ok) {
+ // Store filename early, since it's invalidated by Close.
+ *filename = output_file->filename();
+ ok = output_file->Write(buffer, message_handler);
+ // attempt Close even if write fails.
+ ok &= Close(output_file, message_handler);
+ }
+ if (!ok) {
+ // Clear filename so we end in a consistent state.
+ filename->clear();
+ }
+ return ok;
+}
+
+bool FileSystem::WriteFileAtomic(const StringPiece& filename_sp,
+ const StringPiece& buffer,
+ MessageHandler* message_handler) {
+ const GoogleString filename(filename_sp.as_string());
+ GoogleString tempfilename;
+ return (WriteTempFile(StrCat(filename, ".temp"), buffer, &tempfilename,
+ message_handler) &&
+ RenameFile(tempfilename.c_str(), filename.c_str(), message_handler));
+}
+
+bool FileSystem::Close(File* file, MessageHandler* message_handler) {
+ bool ret = file->Close(message_handler);
+ delete file;
+ return ret;
+}
+
+
+bool FileSystem::RecursivelyMakeDir(const StringPiece& full_path_const,
+ MessageHandler* handler) {
+ bool ret = true;
+ GoogleString full_path = full_path_const.as_string();
+ EnsureEndsInSlash(&full_path);
+ GoogleString subpath;
+ subpath.reserve(full_path.size());
+ size_t old_pos = 0, new_pos;
+ // Note that we intentionally start searching at pos = 1 to avoid having
+ // subpath be "" on absolute paths.
+ while ((new_pos = full_path.find('/', old_pos + 1)) != GoogleString::npos) {
+ // Build up path, one segment at a time.
+ subpath.append(full_path.data() + old_pos, new_pos - old_pos);
+ if (Exists(subpath.c_str(), handler).is_false()) {
+ if (!MakeDir(subpath.c_str(), handler)) {
+ ret = false;
+ break;
+ }
+ } else if (IsDir(subpath.c_str(), handler).is_false()) {
+ handler->Message(kError, "Subpath '%s' of '%s' is a non-directory file.",
+ subpath.c_str(), full_path.c_str());
+ ret = false;
+ break;
+ }
+ old_pos = new_pos;
+ }
+ return ret;
+}
+
+bool FileSystem::RecursiveDirSize(const StringPiece& path, int64* size,
+ MessageHandler* handler) {
+ // TODO(abliss): replace this recursive algorithm with an iterator
+ // that keeps its own state. It can keep a tree of directory names
+ // to save memory, and simplify the implementation of file_cache.Clean.
+ const GoogleString path_string = path.as_string();
+ const char* path_str = path_string.c_str();
+ int64 file_size = 0;
+ StringVector files;
+ if (!ListContents(path_str, &files, handler)) {
+ return false;
+ }
+ const GoogleString prefix = path_string + "/";
+ for (int i = files.size() - 1; i >= 0; i--) {
+ const GoogleString file_name = files[i];
+ BoolOrError isDir = IsDir(file_name.c_str(), handler);
+ if (isDir.is_error()) {
+ return false;
+ } else if (isDir.is_false()) {
+ if (!Size(file_name, &file_size, handler)) {
+ return false;
+ }
+ *size += file_size;
+ } else {
+ // Recurse on directory
+ // TODO(abliss): Should guard against infinite loops here, in
+ // the case of a filesystem with cyclic symlinks.
+ if (!RecursiveDirSize(file_name, size, handler)) {
+ return false;
+ }
+ }
+ }
+ return true;
+}
+
+// Try to make directories to store file.
+void FileSystem::SetupFileDir(const StringPiece& filename,
+ MessageHandler* handler) {
+ size_t last_slash = filename.rfind('/');
+ if (last_slash != StringPiece::npos) {
+ StringPiece directory_name = filename.substr(0, last_slash);
+ if (!RecursivelyMakeDir(directory_name, handler)) {
+ // TODO(sligocki): Specify where dir creation failed?
+ handler->Message(kError, "Could not create directories for file %s",
+ filename.as_string().c_str());
+ }
+ }
+}
+
+} // namespace net_instaweb
Added: httpd/mod_spdy/trunk/net/instaweb/util/file_system_lock_manager.cc
URL: http://svn.apache.org/viewvc/httpd/mod_spdy/trunk/net/instaweb/util/file_system_lock_manager.cc?rev=1591622&view=auto
==============================================================================
--- httpd/mod_spdy/trunk/net/instaweb/util/file_system_lock_manager.cc (added)
+++ httpd/mod_spdy/trunk/net/instaweb/util/file_system_lock_manager.cc Thu May 1 11:43:36 2014
@@ -0,0 +1,109 @@
+/*
+ * Copyright 2010 Google Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+// Author: jmaessen@google.com (Jan Maessen)
+#include "net/instaweb/util/public/file_system_lock_manager.h"
+
+#include "net/instaweb/util/public/basictypes.h"
+#include "net/instaweb/util/public/file_system.h"
+#include "net/instaweb/util/public/scheduler_based_abstract_lock.h"
+#include "net/instaweb/util/public/string.h"
+#include "net/instaweb/util/public/string_util.h"
+
+namespace net_instaweb {
+
+class MessageHandler;
+class Scheduler;
+
+class FileSystemLock : public SchedulerBasedAbstractLock {
+ public:
+ virtual ~FileSystemLock() {
+ if (held_) {
+ Unlock();
+ }
+ }
+
+ virtual bool TryLock() {
+ bool result = false;
+ if (manager_->file_system()->
+ TryLock(name_, manager_->handler()).is_true()) {
+ held_ = result = true;
+ }
+ return result;
+ }
+
+ virtual bool TryLockStealOld(int64 timeout_ms) {
+ bool result = false;
+ if (manager_->file_system()->
+ TryLockWithTimeout(name_, timeout_ms, manager_->handler()).is_true()) {
+ held_ = result = true;
+ }
+ return result;
+ }
+
+ virtual void Unlock() {
+ held_ = !manager_->file_system()->Unlock(name_, manager_->handler());
+ }
+
+ virtual GoogleString name() {
+ return name_;
+ }
+
+ virtual bool Held() {
+ return held_;
+ }
+
+ protected:
+ virtual Scheduler* scheduler() const {
+ return manager_->scheduler();
+ }
+
+ private:
+ friend class FileSystemLockManager;
+
+ // ctor should only be called by CreateNamedLock below.
+ FileSystemLock(const StringPiece& name, FileSystemLockManager* manager)
+ : name_(name.data(), name.size()),
+ manager_(manager),
+ held_(false) { }
+
+ GoogleString name_;
+ FileSystemLockManager* manager_;
+ // The held_ field contains an approximation to whether the lock is locked or
+ // not. If we believe the lock to be locked, we will unlock on destruction.
+ // We therefore try to conservatively leave it "true" when we aren't sure.
+ bool held_;
+
+ DISALLOW_COPY_AND_ASSIGN(FileSystemLock);
+};
+
+FileSystemLockManager::FileSystemLockManager(
+ FileSystem* file_system, const StringPiece& base_path, Scheduler* scheduler,
+ MessageHandler* handler)
+ : file_system_(file_system),
+ base_path_(base_path.as_string()),
+ scheduler_(scheduler),
+ handler_(handler) {
+ EnsureEndsInSlash(&base_path_);
+}
+
+FileSystemLockManager::~FileSystemLockManager() { }
+
+NamedLock* FileSystemLockManager::CreateNamedLock(const StringPiece& name) {
+ return new FileSystemLock(StrCat(base_path_, name), this);
+}
+
+} // namespace net_instaweb
Added: httpd/mod_spdy/trunk/net/instaweb/util/file_system_lock_manager_test.cc
URL: http://svn.apache.org/viewvc/httpd/mod_spdy/trunk/net/instaweb/util/file_system_lock_manager_test.cc?rev=1591622&view=auto
==============================================================================
--- httpd/mod_spdy/trunk/net/instaweb/util/file_system_lock_manager_test.cc (added)
+++ httpd/mod_spdy/trunk/net/instaweb/util/file_system_lock_manager_test.cc Thu May 1 11:43:36 2014
@@ -0,0 +1,271 @@
+/*
+ * Copyright 2010 Google Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+// Author: jmaessen@google.com (Jan Maessen)
+
+// Unit test the file_system_lock_manager using single-threaded mocks.
+
+#include "base/logging.h"
+#include "base/scoped_ptr.h"
+#include "net/instaweb/util/public/basictypes.h"
+#include "net/instaweb/util/public/file_system_lock_manager.h"
+#include "net/instaweb/util/public/google_message_handler.h"
+#include "net/instaweb/util/public/gtest.h"
+#include "net/instaweb/util/public/mem_file_system.h"
+#include "net/instaweb/util/public/mock_scheduler.h"
+#include "net/instaweb/util/public/mock_timer.h"
+#include "net/instaweb/util/public/named_lock_manager.h"
+#include "net/instaweb/util/public/string_util.h"
+#include "net/instaweb/util/public/thread_system.h"
+
+namespace net_instaweb {
+
+namespace {
+
+const char kLock1[] = "lock1";
+const char kLock2[] = "lock2";
+
+const int64 kTimeoutMs = 50000;
+const int64 kWaitMs = 10000;
+
+class FileSystemLockManagerTest : public testing::Test {
+ protected:
+ FileSystemLockManagerTest()
+ : timer_(0),
+ thread_system_(ThreadSystem::CreateThreadSystem()),
+ scheduler_(thread_system_.get(), &timer_),
+ file_system_(thread_system_.get(), &timer_),
+ manager_(&file_system_, GTestTempDir(), &scheduler_, &handler_) { }
+ virtual ~FileSystemLockManagerTest() { }
+
+ NamedLock* MakeLock(const StringPiece& name) {
+ NamedLock* result = manager_.CreateNamedLock(name);
+ CHECK(NULL != result) << "Creating lock " << name;
+ EXPECT_EQ(StrCat(GTestTempDir(), "/", name), result->name());
+ return result;
+ }
+
+ void AllLocksFail(NamedLock* lock) {
+ // Note: we do it in this order to make sure that the timed waits don't
+ // cause the lock to time out.
+ // Note also that we don't do the blocking lock operations, as they'll block
+ // indefinitely here!
+ EXPECT_FALSE(lock->TryLock());
+ EXPECT_FALSE(lock->TryLockStealOld(kTimeoutMs));
+ EXPECT_FALSE(lock->LockTimedWaitStealOld(kWaitMs, kTimeoutMs));
+ EXPECT_FALSE(lock->LockTimedWait(kWaitMs));
+ }
+
+ MockTimer* timer() {
+ return &timer_;
+ }
+
+ MockTimer timer_;
+ scoped_ptr<ThreadSystem> thread_system_;
+ MockScheduler scheduler_;
+ GoogleMessageHandler handler_;
+ MemFileSystem file_system_;
+ FileSystemLockManager manager_;
+
+ private:
+ DISALLOW_COPY_AND_ASSIGN(FileSystemLockManagerTest);
+};
+
+TEST_F(FileSystemLockManagerTest, LockUnlock) {
+ scoped_ptr<NamedLock> lock1(MakeLock(kLock1));
+ // Just do pairs of matched lock / unlock, making sure
+ // we can't lock while the lock is held.
+ EXPECT_TRUE(lock1->TryLock());
+ EXPECT_TRUE(lock1->Held());
+ AllLocksFail(lock1.get());
+
+ lock1->Unlock();
+ EXPECT_FALSE(lock1->Held());
+
+ EXPECT_TRUE(lock1->TryLock());
+ EXPECT_TRUE(lock1->Held());
+ AllLocksFail(lock1.get());
+
+ lock1->Unlock();
+ EXPECT_FALSE(lock1->Held());
+
+ EXPECT_TRUE(lock1->LockTimedWait(kWaitMs));
+ EXPECT_TRUE(lock1->Held());
+ AllLocksFail(lock1.get());
+
+ lock1->Unlock();
+ EXPECT_FALSE(lock1->Held());
+
+ EXPECT_TRUE(lock1->TryLockStealOld(kTimeoutMs));
+ EXPECT_TRUE(lock1->Held());
+ AllLocksFail(lock1.get());
+
+ lock1->Unlock();
+ EXPECT_FALSE(lock1->Held());
+
+ EXPECT_TRUE(lock1->LockTimedWaitStealOld(kWaitMs, kTimeoutMs));
+ EXPECT_TRUE(lock1->Held());
+ AllLocksFail(lock1.get());
+
+ lock1->Unlock();
+ EXPECT_FALSE(lock1->Held());
+}
+
+TEST_F(FileSystemLockManagerTest, DoubleLockUnlock) {
+ scoped_ptr<NamedLock> lock1(MakeLock(kLock1));
+ scoped_ptr<NamedLock> lock11(MakeLock(kLock1));
+ // Just do pairs of matched lock / unlock, but make sure
+ // we hold a separate lock object with the same lock name.
+ EXPECT_TRUE(lock1->TryLock());
+ EXPECT_TRUE(lock1->Held());
+ EXPECT_FALSE(lock11->Held());
+ AllLocksFail(lock11.get());
+ lock1->Unlock();
+ EXPECT_FALSE(lock1->Held());
+ EXPECT_FALSE(lock11->Held());
+
+ EXPECT_TRUE(lock1->TryLock());
+ AllLocksFail(lock11.get());
+ EXPECT_TRUE(lock1->Held());
+ EXPECT_FALSE(lock11->Held());
+ lock1->Unlock();
+ EXPECT_FALSE(lock1->Held());
+ EXPECT_FALSE(lock11->Held());
+
+ EXPECT_TRUE(lock1->LockTimedWait(kWaitMs));
+ EXPECT_TRUE(lock1->Held());
+ EXPECT_FALSE(lock11->Held());
+ AllLocksFail(lock11.get());
+ lock1->Unlock();
+ EXPECT_FALSE(lock1->Held());
+ EXPECT_FALSE(lock11->Held());
+
+ EXPECT_TRUE(lock1->TryLockStealOld(kTimeoutMs));
+ EXPECT_TRUE(lock1->Held());
+ EXPECT_FALSE(lock11->Held());
+ AllLocksFail(lock11.get());
+ lock1->Unlock();
+ EXPECT_FALSE(lock1->Held());
+ EXPECT_FALSE(lock11->Held());
+
+ EXPECT_TRUE(lock1->LockTimedWaitStealOld(kWaitMs, kTimeoutMs));
+ EXPECT_TRUE(lock1->Held());
+ EXPECT_FALSE(lock11->Held());
+ AllLocksFail(lock11.get());
+ lock1->Unlock();
+ EXPECT_FALSE(lock1->Held());
+ EXPECT_FALSE(lock11->Held());
+}
+
+// From this point, we assume all the locking routines hold
+// the lock in equivalent ways. Now we just need to check that
+// their timeout behaviors are correct.
+
+TEST_F(FileSystemLockManagerTest, UnlockOnDestruct) {
+ scoped_ptr<NamedLock> lock1(MakeLock(kLock1));
+ {
+ scoped_ptr<NamedLock> lock11(MakeLock(kLock1));
+ EXPECT_TRUE(lock11->TryLock());
+ EXPECT_FALSE(lock1->TryLock());
+ // Should implicitly unlock on lock11 destructor call.
+ }
+ EXPECT_TRUE(lock1->TryLock());
+}
+
+TEST_F(FileSystemLockManagerTest, LockIndependence) {
+ // Differently-named locks are different.
+ scoped_ptr<NamedLock> lock1(MakeLock(kLock1));
+ scoped_ptr<NamedLock> lock2(MakeLock(kLock2));
+ EXPECT_TRUE(lock1->TryLock());
+ EXPECT_TRUE(lock2->TryLock());
+ EXPECT_FALSE(lock1->TryLock());
+ EXPECT_FALSE(lock2->TryLock());
+ lock2->Unlock();
+ EXPECT_FALSE(lock1->TryLock());
+ EXPECT_TRUE(lock2->TryLock());
+}
+
+TEST_F(FileSystemLockManagerTest, TimeoutFail) {
+ scoped_ptr<NamedLock> lock1(MakeLock(kLock1));
+ EXPECT_TRUE(lock1->TryLock());
+ EXPECT_TRUE(lock1->Held());
+ int64 start_ms = timer()->NowMs();
+ EXPECT_FALSE(lock1->LockTimedWait(kWaitMs));
+ EXPECT_TRUE(lock1->Held()); // was never unlocked...
+ int64 end_ms = timer()->NowMs();
+ EXPECT_LE(start_ms + kWaitMs, end_ms);
+}
+
+TEST_F(FileSystemLockManagerTest, StealOld) {
+ scoped_ptr<NamedLock> lock1(MakeLock(kLock1));
+ EXPECT_TRUE(lock1->TryLock());
+ // Now we can't steal the lock until after >kTimeoutMs has elapsed.
+ EXPECT_FALSE(lock1->TryLockStealOld(kTimeoutMs));
+ timer()->AdvanceMs(kTimeoutMs);
+ EXPECT_FALSE(lock1->TryLockStealOld(kTimeoutMs));
+ // But 1ms longer than kTimeoutMs and we can steal the lock.
+ timer()->AdvanceMs(1);
+ EXPECT_TRUE(lock1->TryLockStealOld(kTimeoutMs));
+ // After steal the timer should reset.
+ EXPECT_FALSE(lock1->TryLockStealOld(kTimeoutMs));
+ timer()->AdvanceMs(kTimeoutMs);
+ EXPECT_FALSE(lock1->TryLockStealOld(kTimeoutMs));
+ EXPECT_TRUE(lock1->Held()); // was never unlocked...
+ // But again expire after >kTimeoutMs elapses.
+ timer()->AdvanceMs(1);
+ EXPECT_TRUE(lock1->TryLockStealOld(kTimeoutMs));
+ EXPECT_TRUE(lock1->Held()); // was never unlocked...
+}
+
+TEST_F(FileSystemLockManagerTest, BlockingStealOld) {
+ scoped_ptr<NamedLock> lock1(MakeLock(kLock1));
+ EXPECT_TRUE(lock1->TryLock());
+ // Now a call to LockTimedWaitStealOld should block until kTimeoutMs has
+ // elapsed.
+ int64 start_ms = timer()->NowMs();
+ lock1->LockTimedWaitStealOld(kTimeoutMs * 100, kTimeoutMs);
+ int64 end_ms = timer()->NowMs();
+ EXPECT_LT(start_ms + kTimeoutMs, end_ms);
+ EXPECT_GT(start_ms + kTimeoutMs * 100, end_ms);
+ // Again the timer should reset after the lock is obtained.
+ EXPECT_FALSE(lock1->TryLockStealOld(kTimeoutMs));
+ timer()->AdvanceMs(kTimeoutMs);
+ EXPECT_FALSE(lock1->TryLockStealOld(kTimeoutMs));
+ timer()->AdvanceMs(1);
+ EXPECT_TRUE(lock1->TryLockStealOld(kTimeoutMs));
+}
+
+TEST_F(FileSystemLockManagerTest, WaitStealOld) {
+ scoped_ptr<NamedLock> lock1(MakeLock(kLock1));
+ EXPECT_TRUE(lock1->TryLock());
+ int64 start_ms = timer()->NowMs();
+ // If we start now, we'll time out with time to spare.
+ EXPECT_FALSE(lock1->LockTimedWaitStealOld(kWaitMs, kTimeoutMs));
+ int64 end_ms = timer()->NowMs();
+ EXPECT_LE(start_ms + kWaitMs, end_ms);
+ EXPECT_GT(start_ms + kTimeoutMs, end_ms);
+ // Advance time so that the lock timeout is within the wait time.
+ int64 time_ms = start_ms + kTimeoutMs - kWaitMs / 2;
+ timer()->SetTimeUs(1000 * time_ms);
+ start_ms = timer()->NowMs();
+ EXPECT_TRUE(lock1->LockTimedWaitStealOld(kWaitMs, kTimeoutMs));
+ end_ms = timer()->NowMs();
+ EXPECT_GT(start_ms + kWaitMs, end_ms);
+}
+
+} // namespace
+
+} // namespace net_instaweb
Added: httpd/mod_spdy/trunk/net/instaweb/util/file_system_test.cc
URL: http://svn.apache.org/viewvc/httpd/mod_spdy/trunk/net/instaweb/util/file_system_test.cc?rev=1591622&view=auto
==============================================================================
--- httpd/mod_spdy/trunk/net/instaweb/util/file_system_test.cc (added)
+++ httpd/mod_spdy/trunk/net/instaweb/util/file_system_test.cc Thu May 1 11:43:36 2014
@@ -0,0 +1,379 @@
+/*
+ * Copyright 2010 Google Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+// Author: jmarantz@google.com (Joshua Marantz)
+
+#include "net/instaweb/util/public/file_system.h"
+
+#include <cstddef>
+#include "net/instaweb/util/public/basictypes.h"
+#include "net/instaweb/util/public/file_system_test.h"
+#include "net/instaweb/util/public/google_message_handler.h"
+#include "net/instaweb/util/public/gtest.h"
+#include "net/instaweb/util/public/string.h"
+#include "net/instaweb/util/public/string_util.h"
+#include "net/instaweb/util/public/timer.h"
+
+namespace net_instaweb {
+
+FileSystemTest::FileSystemTest() { }
+FileSystemTest::~FileSystemTest() { }
+
+GoogleString FileSystemTest::WriteNewFile(const StringPiece& suffix,
+ const GoogleString& content) {
+ GoogleString filename = StrCat(test_tmpdir(), suffix);
+
+ // Make sure we don't read an old file.
+ DeleteRecursively(filename);
+ EXPECT_TRUE(file_system()->WriteFile(filename.c_str(), content, &handler_));
+
+ return filename;
+}
+
+// Check that a file has been read.
+void FileSystemTest::CheckRead(const GoogleString& filename,
+ const GoogleString& expected_contents) {
+ GoogleString buffer;
+ ASSERT_TRUE(file_system()->ReadFile(filename.c_str(), &buffer, &handler_));
+ EXPECT_EQ(buffer, expected_contents);
+}
+
+// Make sure we can no longer read the file by the old name. Note
+// that this will spew some error messages into the log file, and
+// we can add a null_message_handler implementation to
+// swallow them, if they become annoying.
+void FileSystemTest::CheckDoesNotExist(const GoogleString& filename) {
+ GoogleString read_buffer;
+ EXPECT_FALSE(file_system()->ReadFile(filename.c_str(), &read_buffer,
+ &handler_));
+ EXPECT_TRUE(file_system()->Exists(filename.c_str(), &handler_).is_false());
+}
+
+// Write a named file, then read it.
+void FileSystemTest::TestWriteRead() {
+ GoogleString filename = test_tmpdir() + "/write.txt";
+ GoogleString msg("Hello, world!");
+
+ DeleteRecursively(filename);
+ FileSystem::OutputFile* ofile = file_system()->OpenOutputFile(
+ filename.c_str(), &handler_);
+ ASSERT_TRUE(ofile != NULL);
+ EXPECT_TRUE(ofile->Write(msg, &handler_));
+ EXPECT_TRUE(file_system()->Close(ofile, &handler_));
+ CheckRead(filename, msg);
+}
+
+// Write a temp file, then read it.
+void FileSystemTest::TestTemp() {
+ GoogleString prefix = test_tmpdir() + "/temp_prefix";
+ FileSystem::OutputFile* ofile = file_system()->OpenTempFile(
+ prefix, &handler_);
+ ASSERT_TRUE(ofile != NULL);
+ GoogleString filename(ofile->filename());
+ GoogleString msg("Hello, world!");
+ EXPECT_TRUE(ofile->Write(msg, &handler_));
+ EXPECT_TRUE(file_system()->Close(ofile, &handler_));
+
+ CheckRead(filename, msg);
+}
+
+// Write a temp file, rename it, then read it.
+void FileSystemTest::TestRename() {
+ GoogleString from_text = "Now is time time";
+ GoogleString to_file = test_tmpdir() + "/to.txt";
+ DeleteRecursively(to_file);
+
+ GoogleString from_file = WriteNewFile("/from.txt", from_text);
+ ASSERT_TRUE(file_system()->RenameFile(from_file.c_str(), to_file.c_str(),
+ &handler_));
+
+ CheckDoesNotExist(from_file);
+ CheckRead(to_file, from_text);
+}
+
+// Write a file and successfully delete it.
+void FileSystemTest::TestRemove() {
+ GoogleString filename = WriteNewFile("/remove.txt", "Goodbye, world!");
+ ASSERT_TRUE(file_system()->RemoveFile(filename.c_str(), &handler_));
+ CheckDoesNotExist(filename);
+}
+
+// Write a file and check that it exists.
+void FileSystemTest::TestExists() {
+ GoogleString filename = WriteNewFile("/exists.txt", "I'm here.");
+ ASSERT_TRUE(file_system()->Exists(filename.c_str(), &handler_).is_true());
+}
+
+// Create a file along with its directory which does not exist.
+void FileSystemTest::TestCreateFileInDir() {
+ GoogleString dir_name = test_tmpdir() + "/make_dir";
+ DeleteRecursively(dir_name);
+ GoogleString filename = dir_name + "/file-in-dir.txt";
+
+ FileSystem::OutputFile* file =
+ file_system()->OpenOutputFile(filename.c_str(), &handler_);
+ ASSERT_TRUE(file);
+ file_system()->Close(file, &handler_);
+}
+
+
+// Make a directory and check that files may be placed in it.
+void FileSystemTest::TestMakeDir() {
+ GoogleString dir_name = test_tmpdir() + "/make_dir";
+ DeleteRecursively(dir_name);
+ GoogleString filename = dir_name + "/file-in-dir.txt";
+
+ ASSERT_TRUE(file_system()->MakeDir(dir_name.c_str(), &handler_));
+ // ... but we can open a file after we've created the directory.
+ FileSystem::OutputFile* file =
+ file_system()->OpenOutputFile(filename.c_str(), &handler_);
+ ASSERT_TRUE(file);
+ file_system()->Close(file, &handler_);
+}
+
+// Make a directory and check that it is a directory.
+void FileSystemTest::TestIsDir() {
+ GoogleString dir_name = test_tmpdir() + "/this_is_a_dir";
+ DeleteRecursively(dir_name);
+
+ // Make sure we don't think the directory is there when it isn't ...
+ ASSERT_TRUE(file_system()->IsDir(dir_name.c_str(), &handler_).is_false());
+ ASSERT_TRUE(file_system()->MakeDir(dir_name.c_str(), &handler_));
+ // ... and that we do think it's there when it is.
+ ASSERT_TRUE(file_system()->IsDir(dir_name.c_str(), &handler_).is_true());
+
+ // Make sure that we don't think a regular file is a directory.
+ GoogleString filename = dir_name + "/this_is_a_file.txt";
+ GoogleString content = "I'm not a directory.";
+ ASSERT_TRUE(file_system()->WriteFile(filename.c_str(), content, &handler_));
+ ASSERT_TRUE(file_system()->IsDir(filename.c_str(), &handler_).is_false());
+}
+
+// Recursively make directories and check that it worked.
+void FileSystemTest::TestRecursivelyMakeDir() {
+ GoogleString base = test_tmpdir() + "/base";
+ GoogleString long_path = base + "/dir/of/a/really/deep/hierarchy";
+ DeleteRecursively(base);
+
+ // Make sure we don't think the directory is there when it isn't ...
+ ASSERT_TRUE(file_system()->IsDir(long_path.c_str(), &handler_).is_false());
+ ASSERT_TRUE(file_system()->RecursivelyMakeDir(long_path, &handler_));
+ // ... and that we do think it's there when it is.
+ ASSERT_TRUE(file_system()->IsDir(long_path.c_str(), &handler_).is_true());
+}
+
+// Check that we cannot create a directory we do not have permissions for.
+// Note: depends upon root dir not being writable.
+void FileSystemTest::TestRecursivelyMakeDir_NoPermission() {
+ GoogleString base = "/bogus-dir";
+ GoogleString path = base + "/no/permission/to/make/this/dir";
+
+ // Make sure the bogus bottom level directory is not there.
+ ASSERT_TRUE(file_system()->Exists(base.c_str(), &handler_).is_false());
+ // We do not have permission to create it.
+ ASSERT_FALSE(file_system()->RecursivelyMakeDir(path, &handler_));
+}
+
+// Check that we cannot create a directory below a file.
+void FileSystemTest::TestRecursivelyMakeDir_FileInPath() {
+ GoogleString base = test_tmpdir() + "/file-in-path";
+ GoogleString filename = base + "/this-is-a-file";
+ GoogleString bad_path = filename + "/some/more/path";
+ DeleteRecursively(base);
+ GoogleString content = "Your path must end here. You shall not pass!";
+
+ ASSERT_TRUE(file_system()->MakeDir(base.c_str(), &handler_));
+ ASSERT_TRUE(file_system()->WriteFile(filename.c_str(), content, &handler_));
+ ASSERT_FALSE(file_system()->RecursivelyMakeDir(bad_path, &handler_));
+}
+
+void FileSystemTest::TestListContents() {
+ GoogleString dir_name = test_tmpdir() + "/make_dir";
+ DeleteRecursively(dir_name);
+ GoogleString filename1 = dir_name + "/file-in-dir.txt";
+ GoogleString filename2 = dir_name + "/another-file-in-dir.txt";
+ GoogleString content = "Lorem ipsum dolor sit amet";
+
+ StringVector mylist;
+
+ ASSERT_TRUE(file_system()->MakeDir(dir_name.c_str(), &handler_));
+ ASSERT_TRUE(file_system()->WriteFile(filename1.c_str(), content, &handler_));
+ ASSERT_TRUE(file_system()->WriteFile(filename2.c_str(), content, &handler_));
+ EXPECT_TRUE(file_system()->ListContents(dir_name, &mylist, &handler_));
+ EXPECT_EQ(size_t(2), mylist.size());
+ // Make sure our filenames are in there
+ EXPECT_FALSE(filename1.compare(mylist.at(0))
+ && filename1.compare(mylist.at(1)));
+ EXPECT_FALSE(filename2.compare(mylist.at(0))
+ && filename2.compare(mylist.at(1)));
+}
+
+void FileSystemTest::TestAtime() {
+ GoogleString dir_name = test_tmpdir() + "/make_dir";
+ DeleteRecursively(dir_name);
+ GoogleString filename1 = "file-in-dir.txt";
+ GoogleString filename2 = "another-file-in-dir.txt";
+ GoogleString full_path1 = dir_name + "/" + filename1;
+ GoogleString full_path2 = dir_name + "/" + filename2;
+ GoogleString content = "Lorem ipsum dolor sit amet";
+ // We need to sleep a bit between accessing files so that the
+ // difference shows up in in atimes which are measured in seconds.
+ unsigned int sleep_us = 1500000;
+
+ ASSERT_TRUE(file_system()->MakeDir(dir_name.c_str(), &handler_));
+ ASSERT_TRUE(file_system()->WriteFile(full_path1.c_str(), content, &handler_));
+ ASSERT_TRUE(file_system()->WriteFile(full_path2.c_str(), content, &handler_));
+
+ int64 atime1, atime2;
+ CheckRead(full_path1, content);
+ timer()->SleepUs(sleep_us);
+ CheckRead(full_path2, content);
+ ASSERT_TRUE(file_system()->Atime(full_path1, &atime1, &handler_));
+ ASSERT_TRUE(file_system()->Atime(full_path2, &atime2, &handler_));
+ EXPECT_LT(atime1, atime2);
+
+ CheckRead(full_path2, content);
+ timer()->SleepUs(sleep_us);
+ CheckRead(full_path1, content);
+ ASSERT_TRUE(file_system()->Atime(full_path1, &atime1, &handler_));
+ ASSERT_TRUE(file_system()->Atime(full_path2, &atime2, &handler_));
+ EXPECT_LT(atime2, atime1);
+}
+
+void FileSystemTest::TestMtime() {
+ GoogleString dir_name = test_tmpdir() + "/make_dir";
+ DeleteRecursively(dir_name);
+ GoogleString filename1 = "file-in-dir.txt";
+ GoogleString filename2 = "another-file-in-dir.txt";
+ GoogleString full_path1 = dir_name + "/" + filename1;
+ GoogleString full_path2 = dir_name + "/" + filename2;
+ GoogleString content = "Lorem ipsum dolor sit amet";
+ // We need to sleep a bit between accessing files so that the
+ // difference shows up in in atimes which are measured in seconds.
+ unsigned int sleep_us = 1500000;
+
+ // Setup directory to play in.
+ ASSERT_TRUE(file_system()->MakeDir(dir_name.c_str(), &handler_));
+
+ // Write two files with pause between
+ ASSERT_TRUE(file_system()->WriteFile(full_path1.c_str(), content, &handler_));
+ timer()->SleepUs(sleep_us);
+ ASSERT_TRUE(file_system()->WriteFile(full_path2.c_str(), content, &handler_));
+
+ int64 mtime1_orig, mtime2_orig;
+ // Check that File1 was created before File2.
+ ASSERT_TRUE(file_system()->Mtime(full_path1, &mtime1_orig, &handler_));
+ ASSERT_TRUE(file_system()->Mtime(full_path2, &mtime2_orig, &handler_));
+ EXPECT_LT(mtime1_orig, mtime2_orig);
+
+ int64 mtime1_read, mtime2_read;
+ // And that even if you read from File1 later, the C-time is still preserved.
+ timer()->SleepUs(sleep_us);
+ CheckRead(full_path1, content);
+ ASSERT_TRUE(file_system()->Mtime(full_path1, &mtime1_read, &handler_));
+ ASSERT_TRUE(file_system()->Mtime(full_path2, &mtime2_read, &handler_));
+ EXPECT_EQ(mtime1_orig, mtime1_read);
+ EXPECT_EQ(mtime2_orig, mtime2_read);
+
+ int64 mtime1_recreate, mtime2_recreate;
+ // But if we delete File1 and re-create it, the C-time is updated.
+ timer()->SleepUs(sleep_us);
+ ASSERT_TRUE(file_system()->RemoveFile(full_path1.c_str(), &handler_));
+ ASSERT_TRUE(file_system()->WriteFile(full_path1.c_str(), content, &handler_));
+ ASSERT_TRUE(file_system()->Mtime(full_path1, &mtime1_recreate, &handler_));
+ ASSERT_TRUE(file_system()->Mtime(full_path2, &mtime2_recreate, &handler_));
+ EXPECT_LT(mtime1_orig, mtime1_recreate);
+ EXPECT_EQ(mtime2_orig, mtime2_recreate);
+
+ EXPECT_GT(mtime1_recreate, mtime2_recreate);
+}
+
+void FileSystemTest::TestSize() {
+ GoogleString dir_name = test_tmpdir() + "/make_dir";
+ DeleteRecursively(dir_name);
+ GoogleString dir_name2 = dir_name + "/make_dir2";
+ GoogleString filename1 = "file-in-dir.txt";
+ GoogleString filename2 = "another-file-in-dir.txt";
+ GoogleString full_path1 = dir_name2 + "/" + filename1;
+ GoogleString full_path2 = dir_name2 + "/" + filename2;
+ GoogleString content1 = "12345";
+ GoogleString content2 = "1234567890";
+ ASSERT_TRUE(file_system()->MakeDir(dir_name.c_str(), &handler_));
+ ASSERT_TRUE(file_system()->MakeDir(dir_name2.c_str(), &handler_));
+ ASSERT_TRUE(file_system()->WriteFile(full_path1.c_str(),
+ content1, &handler_));
+ ASSERT_TRUE(file_system()->WriteFile(full_path2.c_str(),
+ content2, &handler_));
+ int64 size;
+
+ EXPECT_TRUE(file_system()->Size(full_path1, &size, &handler_));
+ EXPECT_EQ(content1.size(), size_t(size));
+ EXPECT_TRUE(file_system()->Size(full_path2, &size, &handler_));
+ EXPECT_EQ(content2.size(), size_t(size));
+ size = 0;
+ EXPECT_TRUE(file_system()->RecursiveDirSize(dir_name2, &size, &handler_));
+ EXPECT_EQ(content1.size() + content2.size(), size_t(size));
+ size = 0;
+ EXPECT_TRUE(file_system()->RecursiveDirSize(dir_name, &size, &handler_));
+ EXPECT_EQ(content1.size() + content2.size(), size_t(size));
+}
+
+void FileSystemTest::TestLock() {
+ GoogleString dir_name = test_tmpdir() + "/make_dir";
+ DeleteRecursively(dir_name);
+ ASSERT_TRUE(file_system()->MakeDir(dir_name.c_str(), &handler_));
+ GoogleString lock_name = dir_name + "/lock";
+ // Acquire the lock
+ EXPECT_TRUE(file_system()->TryLock(lock_name, &handler_).is_true());
+ // Can't re-acquire the lock
+ EXPECT_TRUE(file_system()->TryLock(lock_name, &handler_).is_false());
+ // Release the lock
+ EXPECT_TRUE(file_system()->Unlock(lock_name, &handler_));
+ // Do it all again to make sure the release worked.
+ EXPECT_TRUE(file_system()->TryLock(lock_name, &handler_).is_true());
+ EXPECT_TRUE(file_system()->TryLock(lock_name, &handler_).is_false());
+ EXPECT_TRUE(file_system()->Unlock(lock_name, &handler_));
+}
+
+// Test lock timeout; assumes the file system has at least 1-second creation
+// granularity.
+void FileSystemTest::TestLockTimeout() {
+ GoogleString dir_name = test_tmpdir() + "/make_dir";
+ DeleteRecursively(dir_name);
+ ASSERT_TRUE(file_system()->MakeDir(dir_name.c_str(), &handler_));
+ GoogleString lock_name = dir_name + "/lock";
+ // Acquire the lock
+ EXPECT_TRUE(file_system()->TryLockWithTimeout(lock_name, Timer::kSecondMs,
+ &handler_).is_true());
+ // Immediate re-acquire should fail. Steal time deliberately long so we don't
+ // steal by mistake (since we're running in non-mock time).
+ EXPECT_TRUE(file_system()->TryLockWithTimeout(lock_name, Timer::kMinuteMs,
+ &handler_).is_false());
+ // Wait 1 second so that we're definitely different from ctime.
+ // Now we should seize lock.
+ timer()->SleepMs(Timer::kSecondMs);
+ EXPECT_TRUE(file_system()->TryLockWithTimeout(lock_name, Timer::kSecondMs,
+ &handler_).is_true());
+ // Lock should still be held.
+ EXPECT_TRUE(file_system()->TryLock(lock_name, &handler_).is_false());
+ EXPECT_TRUE(file_system()->Unlock(lock_name, &handler_));
+ // The result of this second unlock is unknown, but it ought not to crash.
+ file_system()->Unlock(lock_name, &handler_);
+ // Lock should now be unambiguously unlocked.
+ EXPECT_TRUE(file_system()->TryLock(lock_name, &handler_).is_true());
+}
+
+} // namespace net_instaweb
Added: httpd/mod_spdy/trunk/net/instaweb/util/file_writer.cc
URL: http://svn.apache.org/viewvc/httpd/mod_spdy/trunk/net/instaweb/util/file_writer.cc?rev=1591622&view=auto
==============================================================================
--- httpd/mod_spdy/trunk/net/instaweb/util/file_writer.cc (added)
+++ httpd/mod_spdy/trunk/net/instaweb/util/file_writer.cc Thu May 1 11:43:36 2014
@@ -0,0 +1,39 @@
+/*
+ * Copyright 2010 Google Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+// Author: jmarantz@google.com (Joshua Marantz)
+
+#include "net/instaweb/util/public/file_writer.h"
+
+#include "net/instaweb/util/public/file_system.h"
+#include "net/instaweb/util/public/string_util.h"
+
+namespace net_instaweb {
+
+class MessageHandler;
+
+FileWriter::~FileWriter() {
+}
+
+bool FileWriter::Write(const StringPiece& str, MessageHandler* handler) {
+ return file_->Write(str, handler);
+}
+
+bool FileWriter::Flush(MessageHandler* message_handler) {
+ return file_->Flush(message_handler);
+}
+
+} // namespace net_instaweb
Added: httpd/mod_spdy/trunk/net/instaweb/util/filename_encoder.cc
URL: http://svn.apache.org/viewvc/httpd/mod_spdy/trunk/net/instaweb/util/filename_encoder.cc?rev=1591622&view=auto
==============================================================================
--- httpd/mod_spdy/trunk/net/instaweb/util/filename_encoder.cc (added)
+++ httpd/mod_spdy/trunk/net/instaweb/util/filename_encoder.cc Thu May 1 11:43:36 2014
@@ -0,0 +1,43 @@
+/*
+ * Copyright 2010 Google Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+// Author: sligocki@google.com (Shawn Ligocki)
+
+#include "net/instaweb/util/public/filename_encoder.h"
+#include "net/instaweb/util/public/string.h"
+#include "net/instaweb/util/public/string_util.h"
+using net::UrlToFilenameEncoder;
+
+namespace net_instaweb {
+
+FilenameEncoder::~FilenameEncoder() {
+}
+
+void FilenameEncoder::Encode(const StringPiece& filename_prefix,
+ const StringPiece& filename_ending,
+ GoogleString* encoded_filename) {
+ UrlToFilenameEncoder::EncodeSegment(filename_prefix.as_string(),
+ filename_ending.as_string(),
+ '/', encoded_filename);
+}
+
+bool FilenameEncoder::Decode(const StringPiece& encoded_filename,
+ GoogleString* decoded_url) {
+ return UrlToFilenameEncoder::Decode(encoded_filename.as_string(), '/',
+ decoded_url);
+}
+
+} // namespace net_instaweb
Added: httpd/mod_spdy/trunk/net/instaweb/util/filename_encoder_test.cc
URL: http://svn.apache.org/viewvc/httpd/mod_spdy/trunk/net/instaweb/util/filename_encoder_test.cc?rev=1591622&view=auto
==============================================================================
--- httpd/mod_spdy/trunk/net/instaweb/util/filename_encoder_test.cc (added)
+++ httpd/mod_spdy/trunk/net/instaweb/util/filename_encoder_test.cc Thu May 1 11:43:36 2014
@@ -0,0 +1,149 @@
+/*
+ * Copyright 2010 Google Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+// Author: sligocki@google.com (Shawn Ligocki)
+
+#include "net/instaweb/util/public/filename_encoder.h"
+
+#include <cstddef>
+#include "net/instaweb/util/public/basictypes.h"
+#include "net/instaweb/util/public/gtest.h"
+#include "net/instaweb/util/public/string.h"
+#include "net/instaweb/util/public/string_util.h"
+
+namespace {
+// net::kMaximumSubdirectoryLength is defined in url_to_filename_encoder.cc, but
+// we cannot link it.
+const size_t kMaxLen = 128;
+} // namespace
+
+namespace net_instaweb {
+
+// Note -- the exact behavior of the encoder is tested in
+// gfe/tools/loadtesting/spdy_testing/url_to_filename_encoder_test.cc
+//
+// Here we just test that the names meet certain properties:
+// 1. The segments are small
+// 2. The URL can be recovered from the filename
+// 3. No invalid filename characters are present.
+class FilenameEncoderTest : public ::testing::Test {
+ public:
+ FilenameEncoderTest() { }
+
+ protected:
+ void CheckSegmentLength(const StringPiece& escaped_word) {
+ StringPieceVector components;
+ SplitStringPieceToVector(escaped_word, "/", &components, false);
+ for (size_t i = 0; i < components.size(); ++i) {
+ EXPECT_GE(kMaxLen, components[i].size());
+ }
+ }
+
+ void CheckValidChars(const StringPiece& escaped_word) {
+ // These characters are invalid in Windows. We will
+ // ignore / for this test, but add in '.
+ // See http://msdn.microsoft.com/en-us/library/aa365247(VS.85).aspx
+ static const char kInvalidChars[] = "<>:\"\\|?*'";
+ for (size_t i = 0; i < escaped_word.size(); ++i) {
+ char c = escaped_word[i];
+ EXPECT_TRUE(NULL == strchr(kInvalidChars, c));
+ }
+ }
+
+ void Validate(const GoogleString& in_word) {
+ GoogleString escaped_word, decoded_url;
+ encoder.Encode("", in_word, &escaped_word);
+ CheckSegmentLength(escaped_word);
+ CheckValidChars(escaped_word);
+ EXPECT_TRUE(encoder.Decode(escaped_word, &decoded_url));
+ EXPECT_EQ(in_word, decoded_url);
+ }
+
+ FilenameEncoder encoder;
+
+ private:
+ DISALLOW_COPY_AND_ASSIGN(FilenameEncoderTest);
+};
+
+TEST_F(FilenameEncoderTest, DoesNotEscapeAlphanum) {
+ Validate("");
+ Validate("abcdefg");
+ Validate("abcdefghijklmnopqrstuvwxyz");
+ Validate("ZYXWVUT");
+ Validate("ZYXWVUTSRQPONMLKJIHGFEDCBA");
+ Validate("01234567689");
+ Validate("/-_");
+ Validate("abcdefghijklmnopqrstuvwxyzZYXWVUTSRQPONMLKJIHGFEDCBA"
+ "01234567689/-_");
+}
+
+TEST_F(FilenameEncoderTest, DoesEscapeNonAlphanum) {
+ Validate(".");
+ Validate("`~!@#$%^&*()_=+[{]}\\|;:'\",<.>?");
+}
+
+TEST_F(FilenameEncoderTest, DoesEscapeCorrectly) {
+ Validate("index.html");
+ Validate("search?q=dogs&go=&form=QBLH&qs=n");
+ Validate("~joebob/my_neeto-website+with_stuff.asp?id=138&content=true");
+}
+
+TEST_F(FilenameEncoderTest, LongTail) {
+ static char long_word[] =
+ "~joebob/briggs/12345678901234567890123456789012345678901234567890"
+ "1234567890123456789012345678901234567890123456789012345678901234567890"
+ "1234567890123456789012345678901234567890123456789012345678901234567890"
+ "1234567890123456789012345678901234567890123456789012345678901234567890"
+ "1234567890123456789012345678901234567890123456789012345678901234567890"
+ "1234567890123456789012345678901234567890123456789012345678901234567890";
+ Validate(long_word);
+}
+
+TEST_F(FilenameEncoderTest, LongTailDots) {
+ // Here the '.' in the last path segment expands to x2E, making
+ // it hits 128 chars before the input segment gets that big.
+ static char long_word[] =
+ "~joebob/briggs/1234567.1234567.1234567.1234567.1234567."
+ "1234567.1234567.1234567.1234567.1234567.1234567.1234567."
+ "1234567.1234567.1234567.1234567.1234567.1234567.1234567."
+ "1234567.1234567.1234567.1234567.1234567.1234567.1234567."
+ "1234567.1234567.1234567.1234567.1234567.1234567.1234567."
+ "1234567.1234567.1234567.1234567.1234567.1234567.1234567.";
+ Validate(long_word);
+}
+
+TEST_F(FilenameEncoderTest, CornerCasesNearMaxLenNoEscape) {
+ // hit corner cases, +/- 4 characters from kMaxLen
+ for (int i = -4; i <= 4; ++i) {
+ GoogleString input;
+ input.append(i + kMaxLen, 'x');
+ Validate(input);
+ }
+}
+
+TEST_F(FilenameEncoderTest, CornerCasesNearMaxLenWithEscape) {
+ // hit corner cases, +/- 4 characters from kMaxLen. This time we
+ // leave off the last 'x' and put in a '.', which ensures that we
+ // are truncating with '/' *after* the expansion.
+ for (int i = -4; i <= 4; ++i) {
+ GoogleString input;
+ input.append(i + kMaxLen - 1, 'x');
+ input.append(1, '.'); // this will expand to 3 characters.
+ Validate(input);
+ }
+}
+
+} // namespace
Added: httpd/mod_spdy/trunk/net/instaweb/util/function.cc
URL: http://svn.apache.org/viewvc/httpd/mod_spdy/trunk/net/instaweb/util/function.cc?rev=1591622&view=auto
==============================================================================
--- httpd/mod_spdy/trunk/net/instaweb/util/function.cc (added)
+++ httpd/mod_spdy/trunk/net/instaweb/util/function.cc Thu May 1 11:43:36 2014
@@ -0,0 +1,68 @@
+/*
+ * Copyright 2011 Google Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+// Author: jmarantz@google.com (Joshua Marantz)
+
+#include "net/instaweb/util/public/function.h"
+#include "base/logging.h"
+
+namespace net_instaweb {
+
+Function::Function()
+ : quit_requested_(NULL),
+ delete_after_callback_(true) {
+ Reset();
+}
+
+Function::~Function() {
+ DCHECK((run_called_ != cancel_called_) || !delete_after_callback_)
+ << "Either run or cancel should be called";
+}
+
+void Function::Reset() {
+ run_called_ = false;
+ cancel_called_ = false;
+}
+
+void Function::CallRun() {
+ // Note: we need to capture should_delete in a local, since in cases
+ // where it's false, an another thread may be responsible for deleting
+ // us, so it may be unsafe to access our fields after the callback runs
+ // (for example, a different-thread function with a stack-allocated
+ // SchedulerBlockingFunction may exit in between our call to Run() and
+ // where we check for whether to delete or not).
+ bool should_delete = delete_after_callback_;
+ DCHECK(!cancel_called_);
+ DCHECK(!run_called_);
+ run_called_ = true;
+ Run();
+ if (should_delete) {
+ delete this;
+ }
+}
+
+void Function::CallCancel() {
+ bool should_delete = delete_after_callback_;
+ DCHECK(!cancel_called_);
+ DCHECK(!run_called_);
+ cancel_called_ = true;
+ Cancel();
+ if (should_delete) {
+ delete this;
+ }
+}
+
+} // namespace net_instaweb
Added: httpd/mod_spdy/trunk/net/instaweb/util/function_test.cc
URL: http://svn.apache.org/viewvc/httpd/mod_spdy/trunk/net/instaweb/util/function_test.cc?rev=1591622&view=auto
==============================================================================
--- httpd/mod_spdy/trunk/net/instaweb/util/function_test.cc (added)
+++ httpd/mod_spdy/trunk/net/instaweb/util/function_test.cc Thu May 1 11:43:36 2014
@@ -0,0 +1,193 @@
+// Copyright 2011 Google Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+// Author: jmarantz@google.com (Joshua Marantz)
+
+#include "net/instaweb/util/public/function.h"
+#include "net/instaweb/util/public/gtest.h"
+
+namespace {
+
+const char kCharData = 'x';
+const int kIntData = 42;
+const double kDoubleData = 5.5;
+
+} // namespace
+
+namespace net_instaweb {
+
+class FunctionTest : public testing::Test {
+ public:
+ FunctionTest() {
+ Clear();
+ }
+
+ void Clear() {
+ char_ = '\0';
+ int_ = 0;
+ double_ = 0.0;
+ was_run_ = false;
+ was_cancelled_ = false;
+ }
+
+ void Run0() {
+ was_run_ = true;
+ }
+
+ void Run1(char c) {
+ char_ = c;
+ was_run_ = true;
+ }
+
+ void Run2(char c, int i) {
+ char_ = c;
+ int_ = i;
+ was_run_ = true;
+ }
+
+ void Run3(char c, int i, double d) {
+ char_ = c;
+ int_ = i;
+ double_ = d;
+ was_run_ = true;
+ }
+
+ void Cancel() {
+ was_cancelled_ = true;
+ }
+
+ bool Matches(char c, int i, double d) const {
+ return ((c == char_) && (i == int_) && (d == double_));
+ }
+
+ protected:
+ char char_;
+ int int_;
+ double double_;
+ bool was_run_;
+ bool was_cancelled_;
+};
+
+TEST_F(FunctionTest, Run0NoCancel) {
+ FunctionTest* function_test = this;
+ Function* f = MakeFunction(function_test, &FunctionTest::Run0);
+ f->CallRun();
+ EXPECT_TRUE(was_run_);
+ EXPECT_FALSE(was_cancelled_);
+ EXPECT_TRUE(Matches('\0', 0, 0.0));
+}
+
+TEST_F(FunctionTest, Run0NoCancelNoAutoDelete) {
+ FunctionTest* function_test = this;
+ Function* f = MakeFunction(function_test, &FunctionTest::Run0);
+ f->set_delete_after_callback(false);
+ f->CallRun();
+ delete f;
+ EXPECT_TRUE(was_run_);
+ EXPECT_FALSE(was_cancelled_);
+ EXPECT_TRUE(Matches('\0', 0, 0.0));
+}
+
+TEST_F(FunctionTest, Run0WithCancel) {
+ FunctionTest* function_test = this;
+ MakeFunction(function_test, &FunctionTest::Run0,
+ &FunctionTest::Cancel)->CallRun();
+ EXPECT_TRUE(was_run_);
+ EXPECT_FALSE(was_cancelled_);
+ EXPECT_TRUE(Matches('\0', 0, 0.0));
+
+ Clear();
+ MakeFunction(function_test, &FunctionTest::Run0,
+ &FunctionTest::Cancel)->CallCancel();
+ EXPECT_FALSE(was_run_);
+ EXPECT_TRUE(was_cancelled_);
+ EXPECT_TRUE(Matches('\0', 0, 0.0));
+}
+
+TEST_F(FunctionTest, Run1NoCancel) {
+ FunctionTest* function_test = this;
+ MakeFunction(function_test, &FunctionTest::Run1,
+ kCharData)->CallRun();
+ EXPECT_TRUE(was_run_);
+ EXPECT_FALSE(was_cancelled_);
+ EXPECT_TRUE(Matches(kCharData, 0, 0.0));
+}
+
+TEST_F(FunctionTest, Run1WithCancel) {
+ FunctionTest* function_test = this;
+ MakeFunction(function_test, &FunctionTest::Run1, &FunctionTest::Cancel,
+ kCharData)->CallRun();
+ EXPECT_TRUE(was_run_);
+ EXPECT_FALSE(was_cancelled_);
+ EXPECT_TRUE(Matches(kCharData, 0, 0.0));
+
+ Clear();
+ MakeFunction(function_test, &FunctionTest::Run1, &FunctionTest::Cancel,
+ kCharData)->CallCancel();
+ EXPECT_FALSE(was_run_);
+ EXPECT_TRUE(was_cancelled_);
+ EXPECT_TRUE(Matches('\0', 0, 0.0));
+}
+
+TEST_F(FunctionTest, Run2NoCancel) {
+ FunctionTest* function_test = this;
+ MakeFunction(function_test, &FunctionTest::Run2,
+ kCharData, kIntData)->CallRun();
+ EXPECT_TRUE(was_run_);
+ EXPECT_FALSE(was_cancelled_);
+ EXPECT_TRUE(Matches(kCharData, kIntData, 0.0));
+}
+
+TEST_F(FunctionTest, Run2WithCancel) {
+ FunctionTest* function_test = this;
+ MakeFunction(function_test, &FunctionTest::Run2, &FunctionTest::Cancel,
+ kCharData, kIntData)->CallRun();
+ EXPECT_TRUE(was_run_);
+ EXPECT_FALSE(was_cancelled_);
+ EXPECT_TRUE(Matches(kCharData, kIntData, 0.0));
+
+ Clear();
+ MakeFunction(function_test, &FunctionTest::Run2, &FunctionTest::Cancel,
+ kCharData, kIntData)->CallCancel();
+ EXPECT_FALSE(was_run_);
+ EXPECT_TRUE(was_cancelled_);
+ EXPECT_TRUE(Matches('\0', 0, 0.0));
+}
+
+TEST_F(FunctionTest, Run3NoCancel) {
+ FunctionTest* function_test = this;
+ MakeFunction(function_test, &FunctionTest::Run3,
+ kCharData, kIntData, kDoubleData)->CallRun();
+ EXPECT_TRUE(was_run_);
+ EXPECT_FALSE(was_cancelled_);
+ EXPECT_TRUE(Matches(kCharData, kIntData, kDoubleData));
+}
+
+TEST_F(FunctionTest, Run3WithCancel) {
+ FunctionTest* function_test = this;
+ MakeFunction(function_test, &FunctionTest::Run3, &FunctionTest::Cancel,
+ kCharData, kIntData, kDoubleData)->CallRun();
+ EXPECT_TRUE(was_run_);
+ EXPECT_FALSE(was_cancelled_);
+ EXPECT_TRUE(Matches(kCharData, kIntData, kDoubleData));
+
+ Clear();
+ MakeFunction(function_test, &FunctionTest::Run3, &FunctionTest::Cancel,
+ kCharData, kIntData, kDoubleData)->CallCancel();
+ EXPECT_FALSE(was_run_);
+ EXPECT_TRUE(was_cancelled_);
+ EXPECT_TRUE(Matches('\0', 0, 0.0));
+}
+
+} // namespace net_instaweb
Added: httpd/mod_spdy/trunk/net/instaweb/util/google_message_handler.cc
URL: http://svn.apache.org/viewvc/httpd/mod_spdy/trunk/net/instaweb/util/google_message_handler.cc?rev=1591622&view=auto
==============================================================================
--- httpd/mod_spdy/trunk/net/instaweb/util/google_message_handler.cc (added)
+++ httpd/mod_spdy/trunk/net/instaweb/util/google_message_handler.cc Thu May 1 11:43:36 2014
@@ -0,0 +1,76 @@
+/*
+ * Copyright 2010 Google Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+// Author: jmarantz@google.com (Joshua Marantz)
+
+#include "net/instaweb/util/public/google_message_handler.h"
+
+#include <cstdarg>
+#include "base/logging.h"
+#include "net/instaweb/util/public/message_handler.h"
+#include "net/instaweb/util/public/string.h"
+#include "net/instaweb/util/public/string_util.h"
+
+namespace net_instaweb {
+
+void GoogleMessageHandler::MessageVImpl(MessageType type, const char* msg,
+ va_list args) {
+ switch (type) {
+ case kInfo:
+ LOG(INFO) << Format(msg, args);
+ break;
+ case kWarning:
+ LOG(WARNING) << Format(msg, args);
+ break;
+ case kError:
+ LOG(ERROR) << Format(msg, args);
+ break;
+ case kFatal:
+ LOG(FATAL) << Format(msg, args);
+ break;
+ }
+}
+
+void GoogleMessageHandler::FileMessageVImpl(MessageType type, const char* file,
+ int line, const char* msg,
+ va_list args) {
+ switch (type) {
+ case kInfo:
+ LOG(INFO) << file << ":" << line << ": " << Format(msg, args);
+ break;
+ case kWarning:
+ LOG(WARNING) << file << ":" << line << ": " << Format(msg, args);
+ break;
+ case kError:
+ LOG(ERROR) << file << ":" << line << ": " << Format(msg, args);
+ break;
+ case kFatal:
+ LOG(FATAL) << file << ":" << line << ": " << Format(msg, args);
+ break;
+ }
+}
+
+// TODO(sligocki): It'd be nice not to do so much string copying.
+GoogleString GoogleMessageHandler::Format(const char* msg, va_list args) {
+ GoogleString buffer;
+
+ // Ignore the name of this routine: it formats with vsnprintf.
+ // See base/stringprintf.cc.
+ StringAppendV(&buffer, msg, args);
+ return buffer;
+}
+
+} // namespace net_instaweb
Added: httpd/mod_spdy/trunk/net/instaweb/util/google_timer.cc
URL: http://svn.apache.org/viewvc/httpd/mod_spdy/trunk/net/instaweb/util/google_timer.cc?rev=1591622&view=auto
==============================================================================
--- httpd/mod_spdy/trunk/net/instaweb/util/google_timer.cc (added)
+++ httpd/mod_spdy/trunk/net/instaweb/util/google_timer.cc Thu May 1 11:43:36 2014
@@ -0,0 +1,50 @@
+/*
+ * Copyright 2010 Google Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+// Author: jmarantz@google.com (Joshua Marantz)
+
+#include "net/instaweb/util/public/google_timer.h"
+
+#include "net/instaweb/util/public/basictypes.h"
+#include "net/instaweb/util/public/timer.h"
+#include <sys/errno.h>
+#include <sys/time.h>
+#include <unistd.h>
+#include <ctime>
+#include "base/logging.h"
+
+namespace net_instaweb {
+
+GoogleTimer::GoogleTimer() {
+}
+
+GoogleTimer::~GoogleTimer() {
+}
+
+int64 GoogleTimer::NowUs() const {
+ struct timeval tv;
+ struct timezone tz = { 0, 0 }; // UTC
+ if (gettimeofday(&tv, &tz) != 0) {
+ LOG(FATAL) << "Could not determine time of day: " << strerror(errno);
+ }
+ return (tv.tv_sec * 1000000) + tv.tv_usec;
+}
+
+void GoogleTimer::SleepUs(int64 us) {
+ usleep(us);
+}
+
+} // namespace net_instaweb