You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

repair.cc 13KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461
  1. // Copyright (c) 2011 The LevelDB Authors. All rights reserved.
  2. // Use of this source code is governed by a BSD-style license that can be
  3. // found in the LICENSE file. See the AUTHORS file for names of contributors.
  4. //
  5. // We recover the contents of the descriptor from the other files we find.
  6. // (1) Any log files are first converted to tables
  7. // (2) We scan every table to compute
  8. // (a) smallest/largest for the table
  9. // (b) largest sequence number in the table
  10. // (3) We generate descriptor contents:
  11. // - log number is set to zero
  12. // - next-file-number is set to 1 + largest file number we found
  13. // - last-sequence-number is set to largest sequence# found across
  14. // all tables (see 2c)
  15. // - compaction pointers are cleared
  16. // - every table file is added at level 0
  17. //
  18. // Possible optimization 1:
  19. // (a) Compute total size and use to pick appropriate max-level M
  20. // (b) Sort tables by largest sequence# in the table
  21. // (c) For each table: if it overlaps earlier table, place in level-0,
  22. // else place in level-M.
  23. // Possible optimization 2:
  24. // Store per-table metadata (smallest, largest, largest-seq#, ...)
  25. // in the table's meta section to speed up ScanTable.
  26. #include "db/builder.h"
  27. #include "db/db_impl.h"
  28. #include "db/dbformat.h"
  29. #include "db/filename.h"
  30. #include "db/log_reader.h"
  31. #include "db/log_writer.h"
  32. #include "db/memtable.h"
  33. #include "db/table_cache.h"
  34. #include "db/version_edit.h"
  35. #include "db/write_batch_internal.h"
  36. #include "leveldb/comparator.h"
  37. #include "leveldb/db.h"
  38. #include "leveldb/env.h"
  39. namespace leveldb {
  40. namespace {
  41. class Repairer {
  42. public:
  43. Repairer(const std::string& dbname, const Options& options)
  44. : dbname_(dbname),
  45. env_(options.env),
  46. icmp_(options.comparator),
  47. ipolicy_(options.filter_policy),
  48. options_(SanitizeOptions(dbname, &icmp_, &ipolicy_, options)),
  49. owns_info_log_(options_.info_log != options.info_log),
  50. owns_cache_(options_.block_cache != options.block_cache),
  51. next_file_number_(1) {
  52. // TableCache can be small since we expect each table to be opened once.
  53. table_cache_ = new TableCache(dbname_, &options_, 10);
  54. }
  55. ~Repairer() {
  56. delete table_cache_;
  57. if (owns_info_log_) {
  58. delete options_.info_log;
  59. }
  60. if (owns_cache_) {
  61. delete options_.block_cache;
  62. }
  63. }
  64. Status Run() {
  65. Status status = FindFiles();
  66. if (status.ok()) {
  67. ConvertLogFilesToTables();
  68. ExtractMetaData();
  69. status = WriteDescriptor();
  70. }
  71. if (status.ok()) {
  72. unsigned long long bytes = 0;
  73. for (size_t i = 0; i < tables_.size(); i++) {
  74. bytes += tables_[i].meta.file_size;
  75. }
  76. Log(options_.info_log,
  77. "**** Repaired leveldb %s; "
  78. "recovered %d files; %llu bytes. "
  79. "Some data may have been lost. "
  80. "****",
  81. dbname_.c_str(),
  82. static_cast<int>(tables_.size()),
  83. bytes);
  84. }
  85. return status;
  86. }
  87. private:
  88. struct TableInfo {
  89. FileMetaData meta;
  90. SequenceNumber max_sequence;
  91. };
  92. std::string const dbname_;
  93. Env* const env_;
  94. InternalKeyComparator const icmp_;
  95. InternalFilterPolicy const ipolicy_;
  96. Options const options_;
  97. bool owns_info_log_;
  98. bool owns_cache_;
  99. TableCache* table_cache_;
  100. VersionEdit edit_;
  101. std::vector<std::string> manifests_;
  102. std::vector<uint64_t> table_numbers_;
  103. std::vector<uint64_t> logs_;
  104. std::vector<TableInfo> tables_;
  105. uint64_t next_file_number_;
  106. Status FindFiles() {
  107. std::vector<std::string> filenames;
  108. Status status = env_->GetChildren(dbname_, &filenames);
  109. if (!status.ok()) {
  110. return status;
  111. }
  112. if (filenames.empty()) {
  113. return Status::IOError(dbname_, "repair found no files");
  114. }
  115. uint64_t number;
  116. FileType type;
  117. for (size_t i = 0; i < filenames.size(); i++) {
  118. if (ParseFileName(filenames[i], &number, &type)) {
  119. if (type == kDescriptorFile) {
  120. manifests_.push_back(filenames[i]);
  121. } else {
  122. if (number + 1 > next_file_number_) {
  123. next_file_number_ = number + 1;
  124. }
  125. if (type == kLogFile) {
  126. logs_.push_back(number);
  127. } else if (type == kTableFile) {
  128. table_numbers_.push_back(number);
  129. } else {
  130. // Ignore other files
  131. }
  132. }
  133. }
  134. }
  135. return status;
  136. }
  137. void ConvertLogFilesToTables() {
  138. for (size_t i = 0; i < logs_.size(); i++) {
  139. std::string logname = LogFileName(dbname_, logs_[i]);
  140. Status status = ConvertLogToTable(logs_[i]);
  141. if (!status.ok()) {
  142. Log(options_.info_log, "Log #%llu: ignoring conversion error: %s",
  143. (unsigned long long) logs_[i],
  144. status.ToString().c_str());
  145. }
  146. ArchiveFile(logname);
  147. }
  148. }
  149. Status ConvertLogToTable(uint64_t log) {
  150. struct LogReporter : public log::Reader::Reporter {
  151. Env* env;
  152. Logger* info_log;
  153. uint64_t lognum;
  154. virtual void Corruption(size_t bytes, const Status& s) {
  155. // We print error messages for corruption, but continue repairing.
  156. Log(info_log, "Log #%llu: dropping %d bytes; %s",
  157. (unsigned long long) lognum,
  158. static_cast<int>(bytes),
  159. s.ToString().c_str());
  160. }
  161. };
  162. // Open the log file
  163. std::string logname = LogFileName(dbname_, log);
  164. SequentialFile* lfile;
  165. Status status = env_->NewSequentialFile(logname, &lfile);
  166. if (!status.ok()) {
  167. return status;
  168. }
  169. // Create the log reader.
  170. LogReporter reporter;
  171. reporter.env = env_;
  172. reporter.info_log = options_.info_log;
  173. reporter.lognum = log;
  174. // We intentionally make log::Reader do checksumming so that
  175. // corruptions cause entire commits to be skipped instead of
  176. // propagating bad information (like overly large sequence
  177. // numbers).
  178. log::Reader reader(lfile, &reporter, false/*do not checksum*/,
  179. 0/*initial_offset*/);
  180. // Read all the records and add to a memtable
  181. std::string scratch;
  182. Slice record;
  183. WriteBatch batch;
  184. MemTable* mem = new MemTable(icmp_);
  185. mem->Ref();
  186. int counter = 0;
  187. while (reader.ReadRecord(&record, &scratch)) {
  188. if (record.size() < 12) {
  189. reporter.Corruption(
  190. record.size(), Status::Corruption("log record too small", logname));
  191. continue;
  192. }
  193. WriteBatchInternal::SetContents(&batch, record);
  194. status = WriteBatchInternal::InsertInto(&batch, mem);
  195. if (status.ok()) {
  196. counter += WriteBatchInternal::Count(&batch);
  197. } else {
  198. Log(options_.info_log, "Log #%llu: ignoring %s",
  199. (unsigned long long) log,
  200. status.ToString().c_str());
  201. status = Status::OK(); // Keep going with rest of file
  202. }
  203. }
  204. delete lfile;
  205. // Do not record a version edit for this conversion to a Table
  206. // since ExtractMetaData() will also generate edits.
  207. FileMetaData meta;
  208. meta.number = next_file_number_++;
  209. Iterator* iter = mem->NewIterator();
  210. status = BuildTable(dbname_, env_, options_, table_cache_, iter, &meta);
  211. delete iter;
  212. mem->Unref();
  213. mem = NULL;
  214. if (status.ok()) {
  215. if (meta.file_size > 0) {
  216. table_numbers_.push_back(meta.number);
  217. }
  218. }
  219. Log(options_.info_log, "Log #%llu: %d ops saved to Table #%llu %s",
  220. (unsigned long long) log,
  221. counter,
  222. (unsigned long long) meta.number,
  223. status.ToString().c_str());
  224. return status;
  225. }
  226. void ExtractMetaData() {
  227. for (size_t i = 0; i < table_numbers_.size(); i++) {
  228. ScanTable(table_numbers_[i]);
  229. }
  230. }
  231. Iterator* NewTableIterator(const FileMetaData& meta) {
  232. // Same as compaction iterators: if paranoid_checks are on, turn
  233. // on checksum verification.
  234. ReadOptions r;
  235. r.verify_checksums = options_.paranoid_checks;
  236. return table_cache_->NewIterator(r, meta.number, meta.file_size);
  237. }
  238. void ScanTable(uint64_t number) {
  239. TableInfo t;
  240. t.meta.number = number;
  241. std::string fname = TableFileName(dbname_, number);
  242. Status status = env_->GetFileSize(fname, &t.meta.file_size);
  243. if (!status.ok()) {
  244. // Try alternate file name.
  245. fname = SSTTableFileName(dbname_, number);
  246. Status s2 = env_->GetFileSize(fname, &t.meta.file_size);
  247. if (s2.ok()) {
  248. status = Status::OK();
  249. }
  250. }
  251. if (!status.ok()) {
  252. ArchiveFile(TableFileName(dbname_, number));
  253. ArchiveFile(SSTTableFileName(dbname_, number));
  254. Log(options_.info_log, "Table #%llu: dropped: %s",
  255. (unsigned long long) t.meta.number,
  256. status.ToString().c_str());
  257. return;
  258. }
  259. // Extract metadata by scanning through table.
  260. int counter = 0;
  261. Iterator* iter = NewTableIterator(t.meta);
  262. bool empty = true;
  263. ParsedInternalKey parsed;
  264. t.max_sequence = 0;
  265. for (iter->SeekToFirst(); iter->Valid(); iter->Next()) {
  266. Slice key = iter->key();
  267. if (!ParseInternalKey(key, &parsed)) {
  268. Log(options_.info_log, "Table #%llu: unparsable key %s",
  269. (unsigned long long) t.meta.number,
  270. EscapeString(key).c_str());
  271. continue;
  272. }
  273. counter++;
  274. if (empty) {
  275. empty = false;
  276. t.meta.smallest.DecodeFrom(key);
  277. }
  278. t.meta.largest.DecodeFrom(key);
  279. if (parsed.sequence > t.max_sequence) {
  280. t.max_sequence = parsed.sequence;
  281. }
  282. }
  283. if (!iter->status().ok()) {
  284. status = iter->status();
  285. }
  286. delete iter;
  287. Log(options_.info_log, "Table #%llu: %d entries %s",
  288. (unsigned long long) t.meta.number,
  289. counter,
  290. status.ToString().c_str());
  291. if (status.ok()) {
  292. tables_.push_back(t);
  293. } else {
  294. RepairTable(fname, t); // RepairTable archives input file.
  295. }
  296. }
  297. void RepairTable(const std::string& src, TableInfo t) {
  298. // We will copy src contents to a new table and then rename the
  299. // new table over the source.
  300. // Create builder.
  301. std::string copy = TableFileName(dbname_, next_file_number_++);
  302. WritableFile* file;
  303. Status s = env_->NewWritableFile(copy, &file);
  304. if (!s.ok()) {
  305. return;
  306. }
  307. TableBuilder* builder = new TableBuilder(options_, file);
  308. // Copy data.
  309. Iterator* iter = NewTableIterator(t.meta);
  310. int counter = 0;
  311. for (iter->SeekToFirst(); iter->Valid(); iter->Next()) {
  312. builder->Add(iter->key(), iter->value());
  313. counter++;
  314. }
  315. delete iter;
  316. ArchiveFile(src);
  317. if (counter == 0) {
  318. builder->Abandon(); // Nothing to save
  319. } else {
  320. s = builder->Finish();
  321. if (s.ok()) {
  322. t.meta.file_size = builder->FileSize();
  323. }
  324. }
  325. delete builder;
  326. builder = NULL;
  327. if (s.ok()) {
  328. s = file->Close();
  329. }
  330. delete file;
  331. file = NULL;
  332. if (counter > 0 && s.ok()) {
  333. std::string orig = TableFileName(dbname_, t.meta.number);
  334. s = env_->RenameFile(copy, orig);
  335. if (s.ok()) {
  336. Log(options_.info_log, "Table #%llu: %d entries repaired",
  337. (unsigned long long) t.meta.number, counter);
  338. tables_.push_back(t);
  339. }
  340. }
  341. if (!s.ok()) {
  342. env_->DeleteFile(copy);
  343. }
  344. }
  345. Status WriteDescriptor() {
  346. std::string tmp = TempFileName(dbname_, 1);
  347. WritableFile* file;
  348. Status status = env_->NewWritableFile(tmp, &file);
  349. if (!status.ok()) {
  350. return status;
  351. }
  352. SequenceNumber max_sequence = 0;
  353. for (size_t i = 0; i < tables_.size(); i++) {
  354. if (max_sequence < tables_[i].max_sequence) {
  355. max_sequence = tables_[i].max_sequence;
  356. }
  357. }
  358. edit_.SetComparatorName(icmp_.user_comparator()->Name());
  359. edit_.SetLogNumber(0);
  360. edit_.SetNextFile(next_file_number_);
  361. edit_.SetLastSequence(max_sequence);
  362. for (size_t i = 0; i < tables_.size(); i++) {
  363. // TODO(opt): separate out into multiple levels
  364. const TableInfo& t = tables_[i];
  365. edit_.AddFile(0, t.meta.number, t.meta.file_size,
  366. t.meta.smallest, t.meta.largest);
  367. }
  368. //fprintf(stderr, "NewDescriptor:\n%s\n", edit_.DebugString().c_str());
  369. {
  370. log::Writer log(file);
  371. std::string record;
  372. edit_.EncodeTo(&record);
  373. status = log.AddRecord(record);
  374. }
  375. if (status.ok()) {
  376. status = file->Close();
  377. }
  378. delete file;
  379. file = NULL;
  380. if (!status.ok()) {
  381. env_->DeleteFile(tmp);
  382. } else {
  383. // Discard older manifests
  384. for (size_t i = 0; i < manifests_.size(); i++) {
  385. ArchiveFile(dbname_ + "/" + manifests_[i]);
  386. }
  387. // Install new manifest
  388. status = env_->RenameFile(tmp, DescriptorFileName(dbname_, 1));
  389. if (status.ok()) {
  390. status = SetCurrentFile(env_, dbname_, 1);
  391. } else {
  392. env_->DeleteFile(tmp);
  393. }
  394. }
  395. return status;
  396. }
  397. void ArchiveFile(const std::string& fname) {
  398. // Move into another directory. E.g., for
  399. // dir/foo
  400. // rename to
  401. // dir/lost/foo
  402. const char* slash = strrchr(fname.c_str(), '/');
  403. std::string new_dir;
  404. if (slash != NULL) {
  405. new_dir.assign(fname.data(), slash - fname.data());
  406. }
  407. new_dir.append("/lost");
  408. env_->CreateDir(new_dir); // Ignore error
  409. std::string new_file = new_dir;
  410. new_file.append("/");
  411. new_file.append((slash == NULL) ? fname.c_str() : slash + 1);
  412. Status s = env_->RenameFile(fname, new_file);
  413. Log(options_.info_log, "Archiving %s: %s\n",
  414. fname.c_str(), s.ToString().c_str());
  415. }
  416. };
  417. } // namespace
  418. Status RepairDB(const std::string& dbname, const Options& options) {
  419. Repairer repairer(dbname, options);
  420. return repairer.Run();
  421. }
  422. } // namespace leveldb