From 28c3a2080a8d63832179aa8df9de6ece45cf6a4a Mon Sep 17 00:00:00 2001 From: Pieter Wuille Date: Fri, 9 May 2014 18:23:34 +0200 Subject: [PATCH] Squashed 'src/leveldb/' changes from 9094c7f..7924331 42dcc7e Merge upstream LevelDB 1.17. e353fbc Release LevelDB 1.17 269fc6c Release LevelDB 1.16 REVERT: 9094c7f Temporarily revert to writing .sst files instead of .ldb git-subtree-dir: src/leveldb git-subtree-split: 79243314e40ac31d79c68e5658a1d6a64800d50b --- src/leveldb/Makefile | 11 ++++---- src/leveldb/db/filename.cc | 9 ++----- src/leveldb/db/log_reader.cc | 23 +++++++++++------ src/leveldb/db/log_test.cc | 40 +++++++++++++++++++++++++---- src/leveldb/db/repair.cc | 1 - src/leveldb/db/version_set.cc | 14 ---------- src/leveldb/include/leveldb/c.h | 1 - src/leveldb/include/leveldb/db.h | 2 +- src/leveldb/include/leveldb/slice.h | 2 +- 9 files changed, 60 insertions(+), 43 deletions(-) diff --git a/src/leveldb/Makefile b/src/leveldb/Makefile index 344ff2972..f8903b69e 100644 --- a/src/leveldb/Makefile +++ b/src/leveldb/Makefile @@ -72,7 +72,7 @@ SHARED = $(SHARED1) else # Update db.h if you change these. SHARED_MAJOR = 1 -SHARED_MINOR = 15 +SHARED_MINOR = 17 SHARED1 = libleveldb.$(PLATFORM_SHARED_EXT) SHARED2 = $(SHARED1).$(SHARED_MAJOR) SHARED3 = $(SHARED1).$(SHARED_MAJOR).$(SHARED_MINOR) @@ -190,19 +190,20 @@ PLATFORMSROOT=/Applications/Xcode.app/Contents/Developer/Platforms SIMULATORROOT=$(PLATFORMSROOT)/iPhoneSimulator.platform/Developer DEVICEROOT=$(PLATFORMSROOT)/iPhoneOS.platform/Developer IOSVERSION=$(shell defaults read $(PLATFORMSROOT)/iPhoneOS.platform/version CFBundleShortVersionString) +IOSARCH=-arch armv6 -arch armv7 -arch armv7s -arch arm64 .cc.o: mkdir -p ios-x86/$(dir $@) - $(CXX) $(CXXFLAGS) -isysroot $(SIMULATORROOT)/SDKs/iPhoneSimulator$(IOSVERSION).sdk -arch i686 -c $< -o ios-x86/$@ + $(CXX) $(CXXFLAGS) -isysroot $(SIMULATORROOT)/SDKs/iPhoneSimulator$(IOSVERSION).sdk -arch i686 -arch x86_64 -c $< -o ios-x86/$@ mkdir -p ios-arm/$(dir $@) - xcrun -sdk iphoneos $(CXX) $(CXXFLAGS) -isysroot $(DEVICEROOT)/SDKs/iPhoneOS$(IOSVERSION).sdk -arch armv6 -arch armv7 -c $< -o ios-arm/$@ + xcrun -sdk iphoneos $(CXX) $(CXXFLAGS) -isysroot $(DEVICEROOT)/SDKs/iPhoneOS$(IOSVERSION).sdk $(IOSARCH) -c $< -o ios-arm/$@ lipo ios-x86/$@ ios-arm/$@ -create -output $@ .c.o: mkdir -p ios-x86/$(dir $@) - $(CC) $(CFLAGS) -isysroot $(SIMULATORROOT)/SDKs/iPhoneSimulator$(IOSVERSION).sdk -arch i686 -c $< -o ios-x86/$@ + $(CC) $(CFLAGS) -isysroot $(SIMULATORROOT)/SDKs/iPhoneSimulator$(IOSVERSION).sdk -arch i686 -arch x86_64 -c $< -o ios-x86/$@ mkdir -p ios-arm/$(dir $@) - xcrun -sdk iphoneos $(CC) $(CFLAGS) -isysroot $(DEVICEROOT)/SDKs/iPhoneOS$(IOSVERSION).sdk -arch armv6 -arch armv7 -c $< -o ios-arm/$@ + xcrun -sdk iphoneos $(CC) $(CFLAGS) -isysroot $(DEVICEROOT)/SDKs/iPhoneOS$(IOSVERSION).sdk $(IOSARCH) -c $< -o ios-arm/$@ lipo ios-x86/$@ ios-arm/$@ -create -output $@ else diff --git a/src/leveldb/db/filename.cc b/src/leveldb/db/filename.cc index 27d750697..da32946d9 100644 --- a/src/leveldb/db/filename.cc +++ b/src/leveldb/db/filename.cc @@ -29,19 +29,14 @@ std::string LogFileName(const std::string& name, uint64_t number) { return MakeFileName(name, number, "log"); } -// TableFileName returns the filenames we usually write to, while -// SSTTableFileName returns the alternative filenames we also try to read from -// for backward compatibility. For now, swap them around. -// TODO: when compatibility is no longer necessary, swap them back -// (TableFileName to use "ldb" and SSTTableFileName to use "sst"). std::string TableFileName(const std::string& name, uint64_t number) { assert(number > 0); - return MakeFileName(name, number, "sst"); + return MakeFileName(name, number, "ldb"); } std::string SSTTableFileName(const std::string& name, uint64_t number) { assert(number > 0); - return MakeFileName(name, number, "ldb"); + return MakeFileName(name, number, "sst"); } std::string DescriptorFileName(const std::string& dbname, uint64_t number) { diff --git a/src/leveldb/db/log_reader.cc b/src/leveldb/db/log_reader.cc index b35f115aa..4919216d0 100644 --- a/src/leveldb/db/log_reader.cc +++ b/src/leveldb/db/log_reader.cc @@ -133,7 +133,9 @@ bool Reader::ReadRecord(Slice* record, std::string* scratch) { case kEof: if (in_fragmented_record) { - ReportCorruption(scratch->size(), "partial record without end(3)"); + // This can be caused by the writer dying immediately after + // writing a physical record but before completing the next; don't + // treat it as a corruption, just ignore the entire logical record. scratch->clear(); } return false; @@ -193,13 +195,12 @@ unsigned int Reader::ReadPhysicalRecord(Slice* result) { eof_ = true; } continue; - } else if (buffer_.size() == 0) { - // End of file - return kEof; } else { - size_t drop_size = buffer_.size(); + // Note that if buffer_ is non-empty, we have a truncated header at the + // end of the file, which can be caused by the writer crashing in the + // middle of writing the header. Instead of considering this an error, + // just report EOF. buffer_.clear(); - ReportCorruption(drop_size, "truncated record at end of file"); return kEof; } } @@ -213,8 +214,14 @@ unsigned int Reader::ReadPhysicalRecord(Slice* result) { if (kHeaderSize + length > buffer_.size()) { size_t drop_size = buffer_.size(); buffer_.clear(); - ReportCorruption(drop_size, "bad record length"); - return kBadRecord; + if (!eof_) { + ReportCorruption(drop_size, "bad record length"); + return kBadRecord; + } + // If the end of the file has been reached without reading |length| bytes + // of payload, assume the writer died in the middle of writing the record. + // Don't report a corruption. + return kEof; } if (type == kZeroType && length == 0) { diff --git a/src/leveldb/db/log_test.cc b/src/leveldb/db/log_test.cc index 4c5cf8757..91d3caafc 100644 --- a/src/leveldb/db/log_test.cc +++ b/src/leveldb/db/log_test.cc @@ -351,20 +351,32 @@ TEST(LogTest, BadRecordType) { ASSERT_EQ("OK", MatchError("unknown record type")); } -TEST(LogTest, TruncatedTrailingRecord) { +TEST(LogTest, TruncatedTrailingRecordIsIgnored) { Write("foo"); ShrinkSize(4); // Drop all payload as well as a header byte ASSERT_EQ("EOF", Read()); - ASSERT_EQ(kHeaderSize - 1, DroppedBytes()); - ASSERT_EQ("OK", MatchError("truncated record at end of file")); + // Truncated last record is ignored, not treated as an error. + ASSERT_EQ(0, DroppedBytes()); + ASSERT_EQ("", ReportMessage()); } TEST(LogTest, BadLength) { + const int kPayloadSize = kBlockSize - kHeaderSize; + Write(BigString("bar", kPayloadSize)); + Write("foo"); + // Least significant size byte is stored in header[4]. + IncrementByte(4, 1); + ASSERT_EQ("foo", Read()); + ASSERT_EQ(kBlockSize, DroppedBytes()); + ASSERT_EQ("OK", MatchError("bad record length")); +} + +TEST(LogTest, BadLengthAtEndIsIgnored) { Write("foo"); ShrinkSize(1); ASSERT_EQ("EOF", Read()); - ASSERT_EQ(kHeaderSize + 2, DroppedBytes()); - ASSERT_EQ("OK", MatchError("bad record length")); + ASSERT_EQ(0, DroppedBytes()); + ASSERT_EQ("", ReportMessage()); } TEST(LogTest, ChecksumMismatch) { @@ -415,6 +427,24 @@ TEST(LogTest, UnexpectedFirstType) { ASSERT_EQ("OK", MatchError("partial record without end")); } +TEST(LogTest, MissingLastIsIgnored) { + Write(BigString("bar", kBlockSize)); + // Remove the LAST block, including header. + ShrinkSize(14); + ASSERT_EQ("EOF", Read()); + ASSERT_EQ("", ReportMessage()); + ASSERT_EQ(0, DroppedBytes()); +} + +TEST(LogTest, PartialLastIsIgnored) { + Write(BigString("bar", kBlockSize)); + // Cause a bad record length in the LAST block. + ShrinkSize(1); + ASSERT_EQ("EOF", Read()); + ASSERT_EQ("", ReportMessage()); + ASSERT_EQ(0, DroppedBytes()); +} + TEST(LogTest, ErrorJoinsRecords) { // Consider two fragmented records: // first(R1) last(R1) first(R2) last(R2) diff --git a/src/leveldb/db/repair.cc b/src/leveldb/db/repair.cc index 96c9b37af..7727fafc5 100644 --- a/src/leveldb/db/repair.cc +++ b/src/leveldb/db/repair.cc @@ -242,7 +242,6 @@ class Repairer { } void ExtractMetaData() { - std::vector kept; for (size_t i = 0; i < table_numbers_.size(); i++) { ScanTable(table_numbers_[i]); } diff --git a/src/leveldb/db/version_set.cc b/src/leveldb/db/version_set.cc index 517edd3b1..aa83df55e 100644 --- a/src/leveldb/db/version_set.cc +++ b/src/leveldb/db/version_set.cc @@ -54,20 +54,6 @@ static int64_t TotalFileSize(const std::vector& files) { return sum; } -namespace { -std::string IntSetToString(const std::set& s) { - std::string result = "{"; - for (std::set::const_iterator it = s.begin(); - it != s.end(); - ++it) { - result += (result.size() > 1) ? "," : ""; - result += NumberToString(*it); - } - result += "}"; - return result; -} -} // namespace - Version::~Version() { assert(refs_ == 0); diff --git a/src/leveldb/include/leveldb/c.h b/src/leveldb/include/leveldb/c.h index 1fa58866c..1048fe3b8 100644 --- a/src/leveldb/include/leveldb/c.h +++ b/src/leveldb/include/leveldb/c.h @@ -9,7 +9,6 @@ Does not support: . getters for the option types . custom comparators that implement key shortening - . capturing post-write-snapshot . custom iter, db, env, cache implementations using just the C bindings Some conventions: diff --git a/src/leveldb/include/leveldb/db.h b/src/leveldb/include/leveldb/db.h index 5ffb29d52..40851b2aa 100644 --- a/src/leveldb/include/leveldb/db.h +++ b/src/leveldb/include/leveldb/db.h @@ -14,7 +14,7 @@ namespace leveldb { // Update Makefile if you change these static const int kMajorVersion = 1; -static const int kMinorVersion = 15; +static const int kMinorVersion = 17; struct Options; struct ReadOptions; diff --git a/src/leveldb/include/leveldb/slice.h b/src/leveldb/include/leveldb/slice.h index 74ea8fa49..bc367986f 100644 --- a/src/leveldb/include/leveldb/slice.h +++ b/src/leveldb/include/leveldb/slice.h @@ -94,7 +94,7 @@ inline bool operator!=(const Slice& x, const Slice& y) { } inline int Slice::compare(const Slice& b) const { - const int min_len = (size_ < b.size_) ? size_ : b.size_; + const size_t min_len = (size_ < b.size_) ? size_ : b.size_; int r = memcmp(data_, b.data_, min_len); if (r == 0) { if (size_ < b.size_) r = -1;