diff --git a/build_support/cpplint.py b/build_support/cpplint.py deleted file mode 100755 index 221f238f2..000000000 --- a/build_support/cpplint.py +++ /dev/null @@ -1,6603 +0,0 @@ -#!/usr/bin/env python3 -# -# Copyright (c) 2009 Google Inc. All rights reserved. -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions are -# met: -# -# * Redistributions of source code must retain the above copyright -# notice, this list of conditions and the following disclaimer. -# * Redistributions in binary form must reproduce the above -# copyright notice, this list of conditions and the following disclaimer -# in the documentation and/or other materials provided with the -# distribution. -# * Neither the name of Google Inc. nor the names of its -# contributors may be used to endorse or promote products derived from -# this software without specific prior written permission. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -"""Does google-lint on c++ files. - -The goal of this script is to identify places in the code that *may* -be in non-compliance with google style. It does not attempt to fix -up these problems -- the point is to educate. It does also not -attempt to find all problems, or to ensure that everything it does -find is legitimately a problem. - -In particular, we can get very confused by /* and // inside strings! -We do a small hack, which is to ignore //'s with "'s after them on the -same line, but it is far from perfect (in either direction). -""" - -import codecs -import copy -import getopt -import glob -import itertools -import math # for log -import os -import re -import sre_compile -import string -import sys -import sysconfig -import unicodedata -import xml.etree.ElementTree - -# if empty, use defaults -_valid_extensions = set([]) - -__VERSION__ = '1.4.4' - -try: - xrange # Python 2 -except NameError: - # -- pylint: disable=redefined-builtin - xrange = range # Python 3 - - -_USAGE = """ -Syntax: cpplint.py [--verbose=#] [--output=emacs|eclipse|vs7|junit] - [--filter=-x,+y,...] - [--counting=total|toplevel|detailed] [--root=subdir] - [--repository=path] - [--linelength=digits] [--headers=x,y,...] - [--recursive] - [--exclude=path] - [--extensions=hpp,cpp,...] - [--quiet] - [--version] - [file] ... - - Style checker for C/C++ source files. - This is a fork of the Google style checker with minor extensions. - - The style guidelines this tries to follow are those in - https://google.github.io/styleguide/cppguide.html - - Every problem is given a confidence score from 1-5, with 5 meaning we are - certain of the problem, and 1 meaning it could be a legitimate construct. - This will miss some errors, and is not a substitute for a code review. - - To suppress false-positive errors of a certain category, add a - 'NOLINT(category)' comment to the line. NOLINT or NOLINT(*) - suppresses errors of all categories on that line. - - The files passed in will be linted; at least one file must be provided. - Default linted extensions are %s. - Other file types will be ignored. - Change the extensions with the --extensions flag. - - Flags: - - output=emacs|eclipse|vs7|junit - By default, the output is formatted to ease emacs parsing. Visual Studio - compatible output (vs7) may also be used. Further support exists for - eclipse (eclipse), and JUnit (junit). XML parsers such as those used - in Jenkins and Bamboo may also be used. Other formats are unsupported. - - verbose=# - Specify a number 0-5 to restrict errors to certain verbosity levels. - Errors with lower verbosity levels have lower confidence and are more - likely to be false positives. - - quiet - Don't print anything if no errors are found. - - filter=-x,+y,... - Specify a comma-separated list of category-filters to apply: only - error messages whose category names pass the filters will be printed. - (Category names are printed with the message and look like - "[whitespace/indent]".) Filters are evaluated left to right. - "-FOO" and "FOO" means "do not print categories that start with FOO". - "+FOO" means "do print categories that start with FOO". - - Examples: --filter=-whitespace,+whitespace/braces - --filter=whitespace,runtime/printf,+runtime/printf_format - --filter=-,+build/include_what_you_use - - To see a list of all the categories used in cpplint, pass no arg: - --filter= - - counting=total|toplevel|detailed - The total number of errors found is always printed. If - 'toplevel' is provided, then the count of errors in each of - the top-level categories like 'build' and 'whitespace' will - also be printed. If 'detailed' is provided, then a count - is provided for each category like 'build/class'. - - repository=path - The top level directory of the repository, used to derive the header - guard CPP variable. By default, this is determined by searching for a - path that contains .git, .hg, or .svn. When this flag is specified, the - given path is used instead. This option allows the header guard CPP - variable to remain consistent even if members of a team have different - repository root directories (such as when checking out a subdirectory - with SVN). In addition, users of non-mainstream version control systems - can use this flag to ensure readable header guard CPP variables. - - Examples: - Assuming that Alice checks out ProjectName and Bob checks out - ProjectName/trunk and trunk contains src/chrome/ui/browser.h, then - with no --repository flag, the header guard CPP variable will be: - - Alice => TRUNK_SRC_CHROME_BROWSER_UI_BROWSER_H_ - Bob => SRC_CHROME_BROWSER_UI_BROWSER_H_ - - If Alice uses the --repository=trunk flag and Bob omits the flag or - uses --repository=. then the header guard CPP variable will be: - - Alice => SRC_CHROME_BROWSER_UI_BROWSER_H_ - Bob => SRC_CHROME_BROWSER_UI_BROWSER_H_ - - root=subdir - The root directory used for deriving header guard CPP variable. - This directory is relative to the top level directory of the repository - which by default is determined by searching for a directory that contains - .git, .hg, or .svn but can also be controlled with the --repository flag. - If the specified directory does not exist, this flag is ignored. - - Examples: - Assuming that src is the top level directory of the repository (and - cwd=top/src), the header guard CPP variables for - src/chrome/browser/ui/browser.h are: - - No flag => CHROME_BROWSER_UI_BROWSER_H_ - --root=chrome => BROWSER_UI_BROWSER_H_ - --root=chrome/browser => UI_BROWSER_H_ - --root=.. => SRC_CHROME_BROWSER_UI_BROWSER_H_ - - linelength=digits - This is the allowed line length for the project. The default value is - 80 characters. - - Examples: - --linelength=120 - - recursive - Search for files to lint recursively. Each directory given in the list - of files to be linted is replaced by all files that descend from that - directory. Files with extensions not in the valid extensions list are - excluded. - - exclude=path - Exclude the given path from the list of files to be linted. Relative - paths are evaluated relative to the current directory and shell globbing - is performed. This flag can be provided multiple times to exclude - multiple files. - - Examples: - --exclude=one.cc - --exclude=src/*.cc - --exclude=src/*.cc --exclude=test/*.cc - - extensions=extension,extension,... - The allowed file extensions that cpplint will check - - Examples: - --extensions=%s - - headers=x,y,... - The header extensions that cpplint will treat as .h in checks. Values are - automatically added to --extensions list. - (by default, only files with extensions %s will be assumed to be headers) - - Examples: - --headers=%s - --headers=hpp,hxx - --headers=hpp - - cpplint.py supports per-directory configurations specified in CPPLINT.cfg - files. CPPLINT.cfg file can contain a number of key=value pairs. - Currently the following options are supported: - - set noparent - filter=+filter1,-filter2,... - exclude_files=regex - linelength=80 - root=subdir - headers=x,y,... - - "set noparent" option prevents cpplint from traversing directory tree - upwards looking for more .cfg files in parent directories. This option - is usually placed in the top-level project directory. - - The "filter" option is similar in function to --filter flag. It specifies - message filters in addition to the |_DEFAULT_FILTERS| and those specified - through --filter command-line flag. - - "exclude_files" allows to specify a regular expression to be matched against - a file name. If the expression matches, the file is skipped and not run - through the linter. - - "linelength" allows to specify the allowed line length for the project. - - The "root" option is similar in function to the --root flag (see example - above). Paths are relative to the directory of the CPPLINT.cfg. - - The "headers" option is similar in function to the --headers flag - (see example above). - - CPPLINT.cfg has an effect on files in the same directory and all - sub-directories, unless overridden by a nested configuration file. - - Example file: - filter=-build/include_order,+build/include_alpha - exclude_files=.*\\.cc - - The above example disables build/include_order warning and enables - build/include_alpha as well as excludes all .cc from being - processed by linter, in the current directory (where the .cfg - file is located) and all sub-directories. -""" - -# We categorize each error message we print. Here are the categories. -# We want an explicit list so we can list them all in cpplint --filter=. -# If you add a new error message with a new category, add it to the list -# here! cpplint_unittest.py should tell you if you forget to do this. -_ERROR_CATEGORIES = [ - 'build/class', - 'build/c++11', - 'build/c++14', - 'build/c++tr1', - 'build/deprecated', - 'build/endif_comment', - 'build/explicit_make_pair', - 'build/forward_decl', - 'build/header_guard', - 'build/include', - 'build/include_subdir', - 'build/include_alpha', - 'build/include_order', - 'build/include_what_you_use', - 'build/namespaces_literals', - 'build/namespaces', - 'build/printf_format', - 'build/storage_class', - 'legal/copyright', - 'readability/alt_tokens', - 'readability/braces', - 'readability/casting', - 'readability/check', - 'readability/constructors', - 'readability/fn_size', - 'readability/inheritance', - 'readability/multiline_comment', - 'readability/multiline_string', - 'readability/namespace', - 'readability/nolint', - 'readability/nul', - 'readability/strings', - 'readability/todo', - 'readability/utf8', - 'runtime/arrays', - 'runtime/casting', - 'runtime/explicit', - 'runtime/int', - 'runtime/init', - 'runtime/invalid_increment', - 'runtime/member_string_references', - 'runtime/memset', - 'runtime/indentation_namespace', - 'runtime/operator', - 'runtime/printf', - 'runtime/printf_format', - 'runtime/references', - 'runtime/string', - 'runtime/threadsafe_fn', - 'runtime/vlog', - 'whitespace/blank_line', - 'whitespace/braces', - 'whitespace/comma', - 'whitespace/comments', - 'whitespace/empty_conditional_body', - 'whitespace/empty_if_body', - 'whitespace/empty_loop_body', - 'whitespace/end_of_line', - 'whitespace/ending_newline', - 'whitespace/forcolon', - 'whitespace/indent', - 'whitespace/line_length', - 'whitespace/newline', - 'whitespace/operators', - 'whitespace/parens', - 'whitespace/semicolon', - 'whitespace/tab', - 'whitespace/todo', - ] - -# These error categories are no longer enforced by cpplint, but for backwards- -# compatibility they may still appear in NOLINT comments. -_LEGACY_ERROR_CATEGORIES = [ - 'readability/streams', - 'readability/function', - ] - -# The default state of the category filter. This is overridden by the --filter= -# flag. By default all errors are on, so only add here categories that should be -# off by default (i.e., categories that must be enabled by the --filter= flags). -# All entries here should start with a '-' or '+', as in the --filter= flag. -_DEFAULT_FILTERS = ['-build/include_alpha'] - -# The default list of categories suppressed for C (not C++) files. -_DEFAULT_C_SUPPRESSED_CATEGORIES = [ - 'readability/casting', - ] - -# The default list of categories suppressed for Linux Kernel files. -_DEFAULT_KERNEL_SUPPRESSED_CATEGORIES = [ - 'whitespace/tab', - ] - -# We used to check for high-bit characters, but after much discussion we -# decided those were OK, as long as they were in UTF-8 and didn't represent -# hard-coded international strings, which belong in a separate i18n file. - -# C++ headers -_CPP_HEADERS = frozenset([ - # Legacy - 'algobase.h', - 'algo.h', - 'alloc.h', - 'builtinbuf.h', - 'bvector.h', - 'complex.h', - 'defalloc.h', - 'deque.h', - 'editbuf.h', - 'fstream.h', - 'function.h', - 'hash_map', - 'hash_map.h', - 'hash_set', - 'hash_set.h', - 'hashtable.h', - 'heap.h', - 'indstream.h', - 'iomanip.h', - 'iostream.h', - 'istream.h', - 'iterator.h', - 'list.h', - 'map.h', - 'multimap.h', - 'multiset.h', - 'ostream.h', - 'pair.h', - 'parsestream.h', - 'pfstream.h', - 'procbuf.h', - 'pthread_alloc', - 'pthread_alloc.h', - 'rope', - 'rope.h', - 'ropeimpl.h', - 'set.h', - 'slist', - 'slist.h', - 'stack.h', - 'stdiostream.h', - 'stl_alloc.h', - 'stl_relops.h', - 'streambuf.h', - 'stream.h', - 'strfile.h', - 'strstream.h', - 'tempbuf.h', - 'tree.h', - 'type_traits.h', - 'vector.h', - # 17.6.1.2 C++ library headers - 'algorithm', - 'array', - 'atomic', - 'bitset', - 'chrono', - 'codecvt', - 'complex', - 'condition_variable', - 'deque', - 'exception', - 'forward_list', - 'fstream', - 'functional', - 'future', - 'initializer_list', - 'iomanip', - 'ios', - 'iosfwd', - 'iostream', - 'istream', - 'iterator', - 'limits', - 'list', - 'locale', - 'map', - 'memory', - 'mutex', - 'new', - 'numeric', - 'ostream', - 'queue', - 'random', - 'ratio', - 'regex', - 'scoped_allocator', - 'set', - 'sstream', - 'stack', - 'stdexcept', - 'streambuf', - 'string', - 'strstream', - 'system_error', - 'thread', - 'tuple', - 'typeindex', - 'typeinfo', - 'type_traits', - 'unordered_map', - 'unordered_set', - 'utility', - 'valarray', - 'vector', - # 17.6.1.2 C++14 headers - 'shared_mutex', - # 17.6.1.2 C++17 headers - 'any', - 'charconv', - 'codecvt', - 'execution', - 'filesystem', - 'memory_resource', - 'optional', - 'string_view', - 'variant', - # 17.6.1.2 C++ headers for C library facilities - 'cassert', - 'ccomplex', - 'cctype', - 'cerrno', - 'cfenv', - 'cfloat', - 'cinttypes', - 'ciso646', - 'climits', - 'clocale', - 'cmath', - 'csetjmp', - 'csignal', - 'cstdalign', - 'cstdarg', - 'cstdbool', - 'cstddef', - 'cstdint', - 'cstdio', - 'cstdlib', - 'cstring', - 'ctgmath', - 'ctime', - 'cuchar', - 'cwchar', - 'cwctype', - ]) - -# Type names -_TYPES = re.compile( - r'^(?:' - # [dcl.type.simple] - r'(char(16_t|32_t)?)|wchar_t|' - r'bool|short|int|long|signed|unsigned|float|double|' - # [support.types] - r'(ptrdiff_t|size_t|max_align_t|nullptr_t)|' - # [cstdint.syn] - r'(u?int(_fast|_least)?(8|16|32|64)_t)|' - r'(u?int(max|ptr)_t)|' - r')$') - - -# These headers are excluded from [build/include] and [build/include_order] -# checks: -# - Anything not following google file name conventions (containing an -# uppercase character, such as Python.h or nsStringAPI.h, for example). -# - Lua headers. -_THIRD_PARTY_HEADERS_PATTERN = re.compile( - r'^(?:[^/]*[A-Z][^/]*\.h|lua\.h|lauxlib\.h|lualib\.h)$') - -# Pattern for matching FileInfo.BaseName() against test file name -_test_suffixes = ['_test', '_regtest', '_unittest'] -_TEST_FILE_SUFFIX = '(' + '|'.join(_test_suffixes) + r')$' - -# Pattern that matches only complete whitespace, possibly across multiple lines. -_EMPTY_CONDITIONAL_BODY_PATTERN = re.compile(r'^\s*$', re.DOTALL) - -# Assertion macros. These are defined in base/logging.h and -# testing/base/public/gunit.h. -_CHECK_MACROS = [ - 'DCHECK', 'CHECK', - 'EXPECT_TRUE', 'ASSERT_TRUE', - 'EXPECT_FALSE', 'ASSERT_FALSE', - ] - -# Replacement macros for CHECK/DCHECK/EXPECT_TRUE/EXPECT_FALSE -_CHECK_REPLACEMENT = dict([(macro_var, {}) for macro_var in _CHECK_MACROS]) - -for op, replacement in [('==', 'EQ'), ('!=', 'NE'), - ('>=', 'GE'), ('>', 'GT'), - ('<=', 'LE'), ('<', 'LT')]: - _CHECK_REPLACEMENT['DCHECK'][op] = 'DCHECK_%s' % replacement - _CHECK_REPLACEMENT['CHECK'][op] = 'CHECK_%s' % replacement - _CHECK_REPLACEMENT['EXPECT_TRUE'][op] = 'EXPECT_%s' % replacement - _CHECK_REPLACEMENT['ASSERT_TRUE'][op] = 'ASSERT_%s' % replacement - -for op, inv_replacement in [('==', 'NE'), ('!=', 'EQ'), - ('>=', 'LT'), ('>', 'LE'), - ('<=', 'GT'), ('<', 'GE')]: - _CHECK_REPLACEMENT['EXPECT_FALSE'][op] = 'EXPECT_%s' % inv_replacement - _CHECK_REPLACEMENT['ASSERT_FALSE'][op] = 'ASSERT_%s' % inv_replacement - -# Alternative tokens and their replacements. For full list, see section 2.5 -# Alternative tokens [lex.digraph] in the C++ standard. -# -# Digraphs (such as '%:') are not included here since it's a mess to -# match those on a word boundary. -_ALT_TOKEN_REPLACEMENT = { - 'and': '&&', - 'bitor': '|', - 'or': '||', - 'xor': '^', - 'compl': '~', - 'bitand': '&', - 'and_eq': '&=', - 'or_eq': '|=', - 'xor_eq': '^=', - 'not': '!', - 'not_eq': '!=' - } - -# Compile regular expression that matches all the above keywords. The "[ =()]" -# bit is meant to avoid matching these keywords outside of boolean expressions. -# -# False positives include C-style multi-line comments and multi-line strings -# but those have always been troublesome for cpplint. -_ALT_TOKEN_REPLACEMENT_PATTERN = re.compile( - r'[ =()](' + ('|'.join(_ALT_TOKEN_REPLACEMENT.keys())) + r')(?=[ (]|$)') - - -# These constants define types of headers for use with -# _IncludeState.CheckNextIncludeOrder(). -_C_SYS_HEADER = 1 -_CPP_SYS_HEADER = 2 -_LIKELY_MY_HEADER = 3 -_POSSIBLE_MY_HEADER = 4 -_OTHER_HEADER = 5 - -# These constants define the current inline assembly state -_NO_ASM = 0 # Outside of inline assembly block -_INSIDE_ASM = 1 # Inside inline assembly block -_END_ASM = 2 # Last line of inline assembly block -_BLOCK_ASM = 3 # The whole block is an inline assembly block - -# Match start of assembly blocks -_MATCH_ASM = re.compile(r'^\s*(?:asm|_asm|__asm|__asm__)' - r'(?:\s+(volatile|__volatile__))?' - r'\s*[{(]') - -# Match strings that indicate we're working on a C (not C++) file. -_SEARCH_C_FILE = re.compile(r'\b(?:LINT_C_FILE|' - r'vim?:\s*.*(\s*|:)filetype=c(\s*|:|$))') - -# Match string that indicates we're working on a Linux Kernel file. -_SEARCH_KERNEL_FILE = re.compile(r'\b(?:LINT_KERNEL_FILE)') - -_regexp_compile_cache = {} - -# {str, set(int)}: a map from error categories to sets of linenumbers -# on which those errors are expected and should be suppressed. -_error_suppressions = {} - -# The root directory used for deriving header guard CPP variable. -# This is set by --root flag. -_root = None -_root_debug = False - -# The top level repository directory. If set, _root is calculated relative to -# this directory instead of the directory containing version control artifacts. -# This is set by the --repository flag. -_repository = None - -# Files to exclude from linting. This is set by the --exclude flag. -_excludes = None - -# Whether to supress PrintInfo messages -_quiet = False - -# The allowed line length of files. -# This is set by --linelength flag. -_line_length = 80 - -try: - unicode -except NameError: - # -- pylint: disable=redefined-builtin - basestring = unicode = str - -try: - long -except NameError: - # -- pylint: disable=redefined-builtin - long = int - -if sys.version_info < (3,): - # -- pylint: disable=no-member - # BINARY_TYPE = str - itervalues = dict.itervalues - iteritems = dict.iteritems -else: - # BINARY_TYPE = bytes - itervalues = dict.values - iteritems = dict.items - -def unicode_escape_decode(x): - if sys.version_info < (3,): - return codecs.unicode_escape_decode(x)[0] - else: - return x - -# Treat all headers starting with 'h' equally: .h, .hpp, .hxx etc. -# This is set by --headers flag. -_hpp_headers = set([]) - -# {str, bool}: a map from error categories to booleans which indicate if the -# category should be suppressed for every line. -_global_error_suppressions = {} - -def ProcessHppHeadersOption(val): - global _hpp_headers - try: - _hpp_headers = {ext.strip() for ext in val.split(',')} - except ValueError: - PrintUsage('Header extensions must be comma separated list.') - -def IsHeaderExtension(file_extension): - return file_extension in GetHeaderExtensions() - -def GetHeaderExtensions(): - if _hpp_headers: - return _hpp_headers - if _valid_extensions: - return {h for h in _valid_extensions if 'h' in h} - return set(['h', 'hh', 'hpp', 'hxx', 'h++', 'cuh']) - -# The allowed extensions for file names -# This is set by --extensions flag -def GetAllExtensions(): - return GetHeaderExtensions().union(_valid_extensions or set( - ['c', 'cc', 'cpp', 'cxx', 'c++', 'cu'])) - -def ProcessExtensionsOption(val): - global _valid_extensions - try: - extensions = [ext.strip() for ext in val.split(',')] - _valid_extensions = set(extensions) - except ValueError: - PrintUsage('Extensions should be a comma-separated list of values;' - 'for example: extensions=hpp,cpp\n' - 'This could not be parsed: "%s"' % (val,)) - -def GetNonHeaderExtensions(): - return GetAllExtensions().difference(GetHeaderExtensions()) - -def ParseNolintSuppressions(filename, raw_line, linenum, error): - """Updates the global list of line error-suppressions. - - Parses any NOLINT comments on the current line, updating the global - error_suppressions store. Reports an error if the NOLINT comment - was malformed. - - Args: - filename: str, the name of the input file. - raw_line: str, the line of input text, with comments. - linenum: int, the number of the current line. - error: function, an error handler. - """ - matched = Search(r'\bNOLINT(NEXTLINE)?\b(\([^)]+\))?', raw_line) - if matched: - if matched.group(1): - suppressed_line = linenum + 1 - else: - suppressed_line = linenum - category = matched.group(2) - if category in (None, '(*)'): # => "suppress all" - _error_suppressions.setdefault(None, set()).add(suppressed_line) - else: - if category.startswith('(') and category.endswith(')'): - category = category[1:-1] - if category in _ERROR_CATEGORIES: - _error_suppressions.setdefault(category, set()).add(suppressed_line) - elif category not in _LEGACY_ERROR_CATEGORIES: - error(filename, linenum, 'readability/nolint', 5, - 'Unknown NOLINT error category: %s' % category) - - -def ProcessGlobalSuppresions(lines): - """Updates the list of global error suppressions. - - Parses any lint directives in the file that have global effect. - - Args: - lines: An array of strings, each representing a line of the file, with the - last element being empty if the file is terminated with a newline. - """ - for line in lines: - if _SEARCH_C_FILE.search(line): - for category in _DEFAULT_C_SUPPRESSED_CATEGORIES: - _global_error_suppressions[category] = True - if _SEARCH_KERNEL_FILE.search(line): - for category in _DEFAULT_KERNEL_SUPPRESSED_CATEGORIES: - _global_error_suppressions[category] = True - - -def ResetNolintSuppressions(): - """Resets the set of NOLINT suppressions to empty.""" - _error_suppressions.clear() - _global_error_suppressions.clear() - - -def IsErrorSuppressedByNolint(category, linenum): - """Returns true if the specified error category is suppressed on this line. - - Consults the global error_suppressions map populated by - ParseNolintSuppressions/ProcessGlobalSuppresions/ResetNolintSuppressions. - - Args: - category: str, the category of the error. - linenum: int, the current line number. - Returns: - bool, True iff the error should be suppressed due to a NOLINT comment or - global suppression. - """ - return (_global_error_suppressions.get(category, False) or - linenum in _error_suppressions.get(category, set()) or - linenum in _error_suppressions.get(None, set())) - - -def Match(pattern, s): - """Matches the string with the pattern, caching the compiled regexp.""" - # The regexp compilation caching is inlined in both Match and Search for - # performance reasons; factoring it out into a separate function turns out - # to be noticeably expensive. - if pattern not in _regexp_compile_cache: - _regexp_compile_cache[pattern] = sre_compile.compile(pattern) - return _regexp_compile_cache[pattern].match(s) - - -def ReplaceAll(pattern, rep, s): - """Replaces instances of pattern in a string with a replacement. - - The compiled regex is kept in a cache shared by Match and Search. - - Args: - pattern: regex pattern - rep: replacement text - s: search string - - Returns: - string with replacements made (or original string if no replacements) - """ - if pattern not in _regexp_compile_cache: - _regexp_compile_cache[pattern] = sre_compile.compile(pattern) - return _regexp_compile_cache[pattern].sub(rep, s) - - -def Search(pattern, s): - """Searches the string for the pattern, caching the compiled regexp.""" - if pattern not in _regexp_compile_cache: - _regexp_compile_cache[pattern] = sre_compile.compile(pattern) - return _regexp_compile_cache[pattern].search(s) - - -def _IsSourceExtension(s): - """File extension (excluding dot) matches a source file extension.""" - return s in GetNonHeaderExtensions() - - -class _IncludeState(object): - """Tracks line numbers for includes, and the order in which includes appear. - - include_list contains list of lists of (header, line number) pairs. - It's a lists of lists rather than just one flat list to make it - easier to update across preprocessor boundaries. - - Call CheckNextIncludeOrder() once for each header in the file, passing - in the type constants defined above. Calls in an illegal order will - raise an _IncludeError with an appropriate error message. - - """ - # self._section will move monotonically through this set. If it ever - # needs to move backwards, CheckNextIncludeOrder will raise an error. - _INITIAL_SECTION = 0 - _MY_H_SECTION = 1 - _C_SECTION = 2 - _CPP_SECTION = 3 - _OTHER_H_SECTION = 4 - - _TYPE_NAMES = { - _C_SYS_HEADER: 'C system header', - _CPP_SYS_HEADER: 'C++ system header', - _LIKELY_MY_HEADER: 'header this file implements', - _POSSIBLE_MY_HEADER: 'header this file may implement', - _OTHER_HEADER: 'other header', - } - _SECTION_NAMES = { - _INITIAL_SECTION: "... nothing. (This can't be an error.)", - _MY_H_SECTION: 'a header this file implements', - _C_SECTION: 'C system header', - _CPP_SECTION: 'C++ system header', - _OTHER_H_SECTION: 'other header', - } - - def __init__(self): - self.include_list = [[]] - self._section = None - self._last_header = None - self.ResetSection('') - - def FindHeader(self, header): - """Check if a header has already been included. - - Args: - header: header to check. - Returns: - Line number of previous occurrence, or -1 if the header has not - been seen before. - """ - for section_list in self.include_list: - for f in section_list: - if f[0] == header: - return f[1] - return -1 - - def ResetSection(self, directive): - """Reset section checking for preprocessor directive. - - Args: - directive: preprocessor directive (e.g. "if", "else"). - """ - # The name of the current section. - self._section = self._INITIAL_SECTION - # The path of last found header. - self._last_header = '' - - # Update list of includes. Note that we never pop from the - # include list. - if directive in ('if', 'ifdef', 'ifndef'): - self.include_list.append([]) - elif directive in ('else', 'elif'): - self.include_list[-1] = [] - - def SetLastHeader(self, header_path): - self._last_header = header_path - - def CanonicalizeAlphabeticalOrder(self, header_path): - """Returns a path canonicalized for alphabetical comparison. - - - replaces "-" with "_" so they both cmp the same. - - removes '-inl' since we don't require them to be after the main header. - - lowercase everything, just in case. - - Args: - header_path: Path to be canonicalized. - - Returns: - Canonicalized path. - """ - return header_path.replace('-inl.h', '.h').replace('-', '_').lower() - - def IsInAlphabeticalOrder(self, clean_lines, linenum, header_path): - """Check if a header is in alphabetical order with the previous header. - - Args: - clean_lines: A CleansedLines instance containing the file. - linenum: The number of the line to check. - header_path: Canonicalized header to be checked. - - Returns: - Returns true if the header is in alphabetical order. - """ - # If previous section is different from current section, _last_header will - # be reset to empty string, so it's always less than current header. - # - # If previous line was a blank line, assume that the headers are - # intentionally sorted the way they are. - if (self._last_header > header_path and - Match(r'^\s*#\s*include\b', clean_lines.elided[linenum - 1])): - return False - return True - - def CheckNextIncludeOrder(self, header_type): - """Returns a non-empty error message if the next header is out of order. - - This function also updates the internal state to be ready to check - the next include. - - Args: - header_type: One of the _XXX_HEADER constants defined above. - - Returns: - The empty string if the header is in the right order, or an - error message describing what's wrong. - - """ - error_message = ('Found %s after %s' % - (self._TYPE_NAMES[header_type], - self._SECTION_NAMES[self._section])) - - last_section = self._section - - if header_type == _C_SYS_HEADER: - if self._section <= self._C_SECTION: - self._section = self._C_SECTION - else: - self._last_header = '' - return error_message - elif header_type == _CPP_SYS_HEADER: - if self._section <= self._CPP_SECTION: - self._section = self._CPP_SECTION - else: - self._last_header = '' - return error_message - elif header_type == _LIKELY_MY_HEADER: - if self._section <= self._MY_H_SECTION: - self._section = self._MY_H_SECTION - else: - self._section = self._OTHER_H_SECTION - elif header_type == _POSSIBLE_MY_HEADER: - if self._section <= self._MY_H_SECTION: - self._section = self._MY_H_SECTION - else: - # This will always be the fallback because we're not sure - # enough that the header is associated with this file. - self._section = self._OTHER_H_SECTION - else: - assert header_type == _OTHER_HEADER - self._section = self._OTHER_H_SECTION - - if last_section != self._section: - self._last_header = '' - - return '' - - -class _CppLintState(object): - """Maintains module-wide state..""" - - def __init__(self): - self.verbose_level = 1 # global setting. - self.error_count = 0 # global count of reported errors - # filters to apply when emitting error messages - self.filters = _DEFAULT_FILTERS[:] - # backup of filter list. Used to restore the state after each file. - self._filters_backup = self.filters[:] - self.counting = 'total' # In what way are we counting errors? - self.errors_by_category = {} # string to int dict storing error counts - self.quiet = False # Suppress non-error messagess? - - # output format: - # "emacs" - format that emacs can parse (default) - # "eclipse" - format that eclipse can parse - # "vs7" - format that Microsoft Visual Studio 7 can parse - # "junit" - format that Jenkins, Bamboo, etc can parse - self.output_format = 'emacs' - - # For JUnit output, save errors and failures until the end so that they - # can be written into the XML - self._junit_errors = [] - self._junit_failures = [] - - def SetOutputFormat(self, output_format): - """Sets the output format for errors.""" - self.output_format = output_format - - def SetQuiet(self, quiet): - """Sets the module's quiet settings, and returns the previous setting.""" - last_quiet = self.quiet - self.quiet = quiet - return last_quiet - - def SetVerboseLevel(self, level): - """Sets the module's verbosity, and returns the previous setting.""" - last_verbose_level = self.verbose_level - self.verbose_level = level - return last_verbose_level - - def SetCountingStyle(self, counting_style): - """Sets the module's counting options.""" - self.counting = counting_style - - def SetFilters(self, filters): - """Sets the error-message filters. - - These filters are applied when deciding whether to emit a given - error message. - - Args: - filters: A string of comma-separated filters (eg "+whitespace/indent"). - Each filter should start with + or -; else we die. - - Raises: - ValueError: The comma-separated filters did not all start with '+' or '-'. - E.g. "-,+whitespace,-whitespace/indent,whitespace/badfilter" - """ - # Default filters always have less priority than the flag ones. - self.filters = _DEFAULT_FILTERS[:] - self.AddFilters(filters) - - def AddFilters(self, filters): - """ Adds more filters to the existing list of error-message filters. """ - for filt in filters.split(','): - clean_filt = filt.strip() - if clean_filt: - self.filters.append(clean_filt) - for filt in self.filters: - if not (filt.startswith('+') or filt.startswith('-')): - raise ValueError('Every filter in --filters must start with + or -' - ' (%s does not)' % filt) - - def BackupFilters(self): - """ Saves the current filter list to backup storage.""" - self._filters_backup = self.filters[:] - - def RestoreFilters(self): - """ Restores filters previously backed up.""" - self.filters = self._filters_backup[:] - - def ResetErrorCounts(self): - """Sets the module's error statistic back to zero.""" - self.error_count = 0 - self.errors_by_category = {} - - def IncrementErrorCount(self, category): - """Bumps the module's error statistic.""" - self.error_count += 1 - if self.counting in ('toplevel', 'detailed'): - if self.counting != 'detailed': - category = category.split('/')[0] - if category not in self.errors_by_category: - self.errors_by_category[category] = 0 - self.errors_by_category[category] += 1 - - def PrintErrorCounts(self): - """Print a summary of errors by category, and the total.""" - for category, count in sorted(iteritems(self.errors_by_category)): - self.PrintInfo('Category \'%s\' errors found: %d\n' % - (category, count)) - if self.error_count > 0: - self.PrintInfo('Total errors found: %d\n' % self.error_count) - - def PrintInfo(self, message): - if not _quiet and self.output_format != 'junit': - sys.stdout.write(message) - - def PrintError(self, message): - if self.output_format == 'junit': - self._junit_errors.append(message) - else: - sys.stderr.write(message) - - def AddJUnitFailure(self, filename, linenum, message, category, confidence): - self._junit_failures.append((filename, linenum, message, category, - confidence)) - - def FormatJUnitXML(self): - num_errors = len(self._junit_errors) - num_failures = len(self._junit_failures) - - testsuite = xml.etree.ElementTree.Element('testsuite') - testsuite.attrib['name'] = 'cpplint' - testsuite.attrib['errors'] = str(num_errors) - testsuite.attrib['failures'] = str(num_failures) - - if num_errors == 0 and num_failures == 0: - testsuite.attrib['tests'] = str(1) - xml.etree.ElementTree.SubElement(testsuite, 'testcase', name='passed') - - else: - testsuite.attrib['tests'] = str(num_errors + num_failures) - if num_errors > 0: - testcase = xml.etree.ElementTree.SubElement(testsuite, 'testcase') - testcase.attrib['name'] = 'errors' - error = xml.etree.ElementTree.SubElement(testcase, 'error') - error.text = '\n'.join(self._junit_errors) - if num_failures > 0: - # Group failures by file - failed_file_order = [] - failures_by_file = {} - for failure in self._junit_failures: - failed_file = failure[0] - if failed_file not in failed_file_order: - failed_file_order.append(failed_file) - failures_by_file[failed_file] = [] - failures_by_file[failed_file].append(failure) - # Create a testcase for each file - for failed_file in failed_file_order: - failures = failures_by_file[failed_file] - testcase = xml.etree.ElementTree.SubElement(testsuite, 'testcase') - testcase.attrib['name'] = failed_file - failure = xml.etree.ElementTree.SubElement(testcase, 'failure') - template = '{0}: {1} [{2}] [{3}]' - texts = [template.format(f[1], f[2], f[3], f[4]) for f in failures] - failure.text = '\n'.join(texts) - - xml_decl = '\n' - return xml_decl + xml.etree.ElementTree.tostring(testsuite, 'utf-8').decode('utf-8') - - -_cpplint_state = _CppLintState() - - -def _OutputFormat(): - """Gets the module's output format.""" - return _cpplint_state.output_format - - -def _SetOutputFormat(output_format): - """Sets the module's output format.""" - _cpplint_state.SetOutputFormat(output_format) - -def _Quiet(): - """Return's the module's quiet setting.""" - return _cpplint_state.quiet - -def _SetQuiet(quiet): - """Set the module's quiet status, and return previous setting.""" - return _cpplint_state.SetQuiet(quiet) - - -def _VerboseLevel(): - """Returns the module's verbosity setting.""" - return _cpplint_state.verbose_level - - -def _SetVerboseLevel(level): - """Sets the module's verbosity, and returns the previous setting.""" - return _cpplint_state.SetVerboseLevel(level) - - -def _SetCountingStyle(level): - """Sets the module's counting options.""" - _cpplint_state.SetCountingStyle(level) - - -def _Filters(): - """Returns the module's list of output filters, as a list.""" - return _cpplint_state.filters - - -def _SetFilters(filters): - """Sets the module's error-message filters. - - These filters are applied when deciding whether to emit a given - error message. - - Args: - filters: A string of comma-separated filters (eg "whitespace/indent"). - Each filter should start with + or -; else we die. - """ - _cpplint_state.SetFilters(filters) - -def _AddFilters(filters): - """Adds more filter overrides. - - Unlike _SetFilters, this function does not reset the current list of filters - available. - - Args: - filters: A string of comma-separated filters (eg "whitespace/indent"). - Each filter should start with + or -; else we die. - """ - _cpplint_state.AddFilters(filters) - -def _BackupFilters(): - """ Saves the current filter list to backup storage.""" - _cpplint_state.BackupFilters() - -def _RestoreFilters(): - """ Restores filters previously backed up.""" - _cpplint_state.RestoreFilters() - -class _FunctionState(object): - """Tracks current function name and the number of lines in its body.""" - - _NORMAL_TRIGGER = 250 # for --v=0, 500 for --v=1, etc. - _TEST_TRIGGER = 400 # about 50% more than _NORMAL_TRIGGER. - - def __init__(self): - self.in_a_function = False - self.lines_in_function = 0 - self.current_function = '' - - def Begin(self, function_name): - """Start analyzing function body. - - Args: - function_name: The name of the function being tracked. - """ - self.in_a_function = True - self.lines_in_function = 0 - self.current_function = function_name - - def Count(self): - """Count line in current function body.""" - if self.in_a_function: - self.lines_in_function += 1 - - def Check(self, error, filename, linenum): - """Report if too many lines in function body. - - Args: - error: The function to call with any errors found. - filename: The name of the current file. - linenum: The number of the line to check. - """ - if not self.in_a_function: - return - - if Match(r'T(EST|est)', self.current_function): - base_trigger = self._TEST_TRIGGER - else: - base_trigger = self._NORMAL_TRIGGER - trigger = base_trigger * 2**_VerboseLevel() - - if self.lines_in_function > trigger: - error_level = int(math.log(self.lines_in_function / base_trigger, 2)) - # 50 => 0, 100 => 1, 200 => 2, 400 => 3, 800 => 4, 1600 => 5, ... - if error_level > 5: - error_level = 5 - error(filename, linenum, 'readability/fn_size', error_level, - 'Small and focused functions are preferred:' - ' %s has %d non-comment lines' - ' (error triggered by exceeding %d lines).' % ( - self.current_function, self.lines_in_function, trigger)) - - def End(self): - """Stop analyzing function body.""" - self.in_a_function = False - - -class _IncludeError(Exception): - """Indicates a problem with the include order in a file.""" - pass - - -class FileInfo(object): - """Provides utility functions for filenames. - - FileInfo provides easy access to the components of a file's path - relative to the project root. - """ - - def __init__(self, filename): - self._filename = filename - - def FullName(self): - """Make Windows paths like Unix.""" - return os.path.abspath(self._filename).replace('\\', '/') - - def RepositoryName(self): - r"""FullName after removing the local path to the repository. - - If we have a real absolute path name here we can try to do something smart: - detecting the root of the checkout and truncating /path/to/checkout from - the name so that we get header guards that don't include things like - "C:\\Documents and Settings\\..." or "/home/username/..." in them and thus - people on different computers who have checked the source out to different - locations won't see bogus errors. - """ - fullname = self.FullName() - - if os.path.exists(fullname): - project_dir = os.path.dirname(fullname) - - # If the user specified a repository path, it exists, and the file is - # contained in it, use the specified repository path - if _repository: - repo = FileInfo(_repository).FullName() - root_dir = project_dir - while os.path.exists(root_dir): - # allow case insensitive compare on Windows - if os.path.normcase(root_dir) == os.path.normcase(repo): - return os.path.relpath(fullname, root_dir).replace('\\', '/') - one_up_dir = os.path.dirname(root_dir) - if one_up_dir == root_dir: - break - root_dir = one_up_dir - - if os.path.exists(os.path.join(project_dir, ".svn")): - # If there's a .svn file in the current directory, we recursively look - # up the directory tree for the top of the SVN checkout - root_dir = project_dir - one_up_dir = os.path.dirname(root_dir) - while os.path.exists(os.path.join(one_up_dir, ".svn")): - root_dir = os.path.dirname(root_dir) - one_up_dir = os.path.dirname(one_up_dir) - - prefix = os.path.commonprefix([root_dir, project_dir]) - return fullname[len(prefix) + 1:] - - # Not SVN <= 1.6? Try to find a git, hg, or svn top level directory by - # searching up from the current path. - root_dir = current_dir = os.path.dirname(fullname) - while current_dir != os.path.dirname(current_dir): - if (os.path.exists(os.path.join(current_dir, ".git")) or - os.path.exists(os.path.join(current_dir, ".hg")) or - os.path.exists(os.path.join(current_dir, ".svn"))): - root_dir = current_dir - current_dir = os.path.dirname(current_dir) - - if (os.path.exists(os.path.join(root_dir, ".git")) or - os.path.exists(os.path.join(root_dir, ".hg")) or - os.path.exists(os.path.join(root_dir, ".svn"))): - prefix = os.path.commonprefix([root_dir, project_dir]) - return fullname[len(prefix) + 1:] - - # Don't know what to do; header guard warnings may be wrong... - return fullname - - def Split(self): - """Splits the file into the directory, basename, and extension. - - For 'chrome/browser/browser.cc', Split() would - return ('chrome/browser', 'browser', '.cc') - - Returns: - A tuple of (directory, basename, extension). - """ - - googlename = self.RepositoryName() - project, rest = os.path.split(googlename) - return (project,) + os.path.splitext(rest) - - def BaseName(self): - """File base name - text after the final slash, before the final period.""" - return self.Split()[1] - - def Extension(self): - """File extension - text following the final period, includes that period.""" - return self.Split()[2] - - def NoExtension(self): - """File has no source file extension.""" - return '/'.join(self.Split()[0:2]) - - def IsSource(self): - """File has a source file extension.""" - return _IsSourceExtension(self.Extension()[1:]) - - -def _ShouldPrintError(category, confidence, linenum): - """If confidence >= verbose, category passes filter and is not suppressed.""" - - # There are three ways we might decide not to print an error message: - # a "NOLINT(category)" comment appears in the source, - # the verbosity level isn't high enough, or the filters filter it out. - if IsErrorSuppressedByNolint(category, linenum): - return False - - if confidence < _cpplint_state.verbose_level: - return False - - is_filtered = False - for one_filter in _Filters(): - if one_filter.startswith('-'): - if category.startswith(one_filter[1:]): - is_filtered = True - elif one_filter.startswith('+'): - if category.startswith(one_filter[1:]): - is_filtered = False - else: - assert False # should have been checked for in SetFilter. - if is_filtered: - return False - - return True - - -def Error(filename, linenum, category, confidence, message): - """Logs the fact we've found a lint error. - - We log where the error was found, and also our confidence in the error, - that is, how certain we are this is a legitimate style regression, and - not a misidentification or a use that's sometimes justified. - - False positives can be suppressed by the use of - "cpplint(category)" comments on the offending line. These are - parsed into _error_suppressions. - - Args: - filename: The name of the file containing the error. - linenum: The number of the line containing the error. - category: A string used to describe the "category" this bug - falls under: "whitespace", say, or "runtime". Categories - may have a hierarchy separated by slashes: "whitespace/indent". - confidence: A number from 1-5 representing a confidence score for - the error, with 5 meaning that we are certain of the problem, - and 1 meaning that it could be a legitimate construct. - message: The error message. - """ - if _ShouldPrintError(category, confidence, linenum): - _cpplint_state.IncrementErrorCount(category) - if _cpplint_state.output_format == 'vs7': - _cpplint_state.PrintError('%s(%s): error cpplint: [%s] %s [%d]\n' % ( - filename, linenum, category, message, confidence)) - elif _cpplint_state.output_format == 'eclipse': - sys.stderr.write('%s:%s: warning: %s [%s] [%d]\n' % ( - filename, linenum, message, category, confidence)) - elif _cpplint_state.output_format == 'junit': - _cpplint_state.AddJUnitFailure(filename, linenum, message, category, - confidence) - else: - final_message = '%s:%s: %s [%s] [%d]\n' % ( - filename, linenum, message, category, confidence) - sys.stderr.write(final_message) - -# Matches standard C++ escape sequences per 2.13.2.3 of the C++ standard. -_RE_PATTERN_CLEANSE_LINE_ESCAPES = re.compile( - r'\\([abfnrtv?"\\\']|\d+|x[0-9a-fA-F]+)') -# Match a single C style comment on the same line. -_RE_PATTERN_C_COMMENTS = r'/\*(?:[^*]|\*(?!/))*\*/' -# Matches multi-line C style comments. -# This RE is a little bit more complicated than one might expect, because we -# have to take care of space removals tools so we can handle comments inside -# statements better. -# The current rule is: We only clear spaces from both sides when we're at the -# end of the line. Otherwise, we try to remove spaces from the right side, -# if this doesn't work we try on left side but only if there's a non-character -# on the right. -_RE_PATTERN_CLEANSE_LINE_C_COMMENTS = re.compile( - r'(\s*' + _RE_PATTERN_C_COMMENTS + r'\s*$|' + - _RE_PATTERN_C_COMMENTS + r'\s+|' + - r'\s+' + _RE_PATTERN_C_COMMENTS + r'(?=\W)|' + - _RE_PATTERN_C_COMMENTS + r')') - - -def IsCppString(line): - """Does line terminate so, that the next symbol is in string constant. - - This function does not consider single-line nor multi-line comments. - - Args: - line: is a partial line of code starting from the 0..n. - - Returns: - True, if next character appended to 'line' is inside a - string constant. - """ - - line = line.replace(r'\\', 'XX') # after this, \\" does not match to \" - return ((line.count('"') - line.count(r'\"') - line.count("'\"'")) & 1) == 1 - - -def CleanseRawStrings(raw_lines): - """Removes C++11 raw strings from lines. - - Before: - static const char kData[] = R"( - multi-line string - )"; - - After: - static const char kData[] = "" - (replaced by blank line) - ""; - - Args: - raw_lines: list of raw lines. - - Returns: - list of lines with C++11 raw strings replaced by empty strings. - """ - - delimiter = None - lines_without_raw_strings = [] - for line in raw_lines: - if delimiter: - # Inside a raw string, look for the end - end = line.find(delimiter) - if end >= 0: - # Found the end of the string, match leading space for this - # line and resume copying the original lines, and also insert - # a "" on the last line. - leading_space = Match(r'^(\s*)\S', line) - line = leading_space.group(1) + '""' + line[end + len(delimiter):] - delimiter = None - else: - # Haven't found the end yet, append a blank line. - line = '""' - - # Look for beginning of a raw string, and replace them with - # empty strings. This is done in a loop to handle multiple raw - # strings on the same line. - while delimiter is None: - # Look for beginning of a raw string. - # See 2.14.15 [lex.string] for syntax. - # - # Once we have matched a raw string, we check the prefix of the - # line to make sure that the line is not part of a single line - # comment. It's done this way because we remove raw strings - # before removing comments as opposed to removing comments - # before removing raw strings. This is because there are some - # cpplint checks that requires the comments to be preserved, but - # we don't want to check comments that are inside raw strings. - matched = Match(r'^(.*?)\b(?:R|u8R|uR|UR|LR)"([^\s\\()]*)\((.*)$', line) - if (matched and - not Match(r'^([^\'"]|\'(\\.|[^\'])*\'|"(\\.|[^"])*")*//', - matched.group(1))): - delimiter = ')' + matched.group(2) + '"' - - end = matched.group(3).find(delimiter) - if end >= 0: - # Raw string ended on same line - line = (matched.group(1) + '""' + - matched.group(3)[end + len(delimiter):]) - delimiter = None - else: - # Start of a multi-line raw string - line = matched.group(1) + '""' - else: - break - - lines_without_raw_strings.append(line) - - # TODO(unknown): if delimiter is not None here, we might want to - # emit a warning for unterminated string. - return lines_without_raw_strings - - -def FindNextMultiLineCommentStart(lines, lineix): - """Find the beginning marker for a multiline comment.""" - while lineix < len(lines): - if lines[lineix].strip().startswith('/*'): - # Only return this marker if the comment goes beyond this line - if lines[lineix].strip().find('*/', 2) < 0: - return lineix - lineix += 1 - return len(lines) - - -def FindNextMultiLineCommentEnd(lines, lineix): - """We are inside a comment, find the end marker.""" - while lineix < len(lines): - if lines[lineix].strip().endswith('*/'): - return lineix - lineix += 1 - return len(lines) - - -def RemoveMultiLineCommentsFromRange(lines, begin, end): - """Clears a range of lines for multi-line comments.""" - # Having // dummy comments makes the lines non-empty, so we will not get - # unnecessary blank line warnings later in the code. - for i in range(begin, end): - lines[i] = '/**/' - - -def RemoveMultiLineComments(filename, lines, error): - """Removes multiline (c-style) comments from lines.""" - lineix = 0 - while lineix < len(lines): - lineix_begin = FindNextMultiLineCommentStart(lines, lineix) - if lineix_begin >= len(lines): - return - lineix_end = FindNextMultiLineCommentEnd(lines, lineix_begin) - if lineix_end >= len(lines): - error(filename, lineix_begin + 1, 'readability/multiline_comment', 5, - 'Could not find end of multi-line comment') - return - RemoveMultiLineCommentsFromRange(lines, lineix_begin, lineix_end + 1) - lineix = lineix_end + 1 - - -def CleanseComments(line): - """Removes //-comments and single-line C-style /* */ comments. - - Args: - line: A line of C++ source. - - Returns: - The line with single-line comments removed. - """ - commentpos = line.find('//') - if commentpos != -1 and not IsCppString(line[:commentpos]): - line = line[:commentpos].rstrip() - # get rid of /* ... */ - return _RE_PATTERN_CLEANSE_LINE_C_COMMENTS.sub('', line) - - -class CleansedLines(object): - """Holds 4 copies of all lines with different preprocessing applied to them. - - 1) elided member contains lines without strings and comments. - 2) lines member contains lines without comments. - 3) raw_lines member contains all the lines without processing. - 4) lines_without_raw_strings member is same as raw_lines, but with C++11 raw - strings removed. - All these members are of , and of the same length. - """ - - def __init__(self, lines): - self.elided = [] - self.lines = [] - self.raw_lines = lines - self.num_lines = len(lines) - self.lines_without_raw_strings = CleanseRawStrings(lines) - for linenum in range(len(self.lines_without_raw_strings)): - self.lines.append(CleanseComments( - self.lines_without_raw_strings[linenum])) - elided = self._CollapseStrings(self.lines_without_raw_strings[linenum]) - self.elided.append(CleanseComments(elided)) - - def NumLines(self): - """Returns the number of lines represented.""" - return self.num_lines - - @staticmethod - def _CollapseStrings(elided): - """Collapses strings and chars on a line to simple "" or '' blocks. - - We nix strings first so we're not fooled by text like '"http://"' - - Args: - elided: The line being processed. - - Returns: - The line with collapsed strings. - """ - if _RE_PATTERN_INCLUDE.match(elided): - return elided - - # Remove escaped characters first to make quote/single quote collapsing - # basic. Things that look like escaped characters shouldn't occur - # outside of strings and chars. - elided = _RE_PATTERN_CLEANSE_LINE_ESCAPES.sub('', elided) - - # Replace quoted strings and digit separators. Both single quotes - # and double quotes are processed in the same loop, otherwise - # nested quotes wouldn't work. - collapsed = '' - while True: - # Find the first quote character - match = Match(r'^([^\'"]*)([\'"])(.*)$', elided) - if not match: - collapsed += elided - break - head, quote, tail = match.groups() - - if quote == '"': - # Collapse double quoted strings - second_quote = tail.find('"') - if second_quote >= 0: - collapsed += head + '""' - elided = tail[second_quote + 1:] - else: - # Unmatched double quote, don't bother processing the rest - # of the line since this is probably a multiline string. - collapsed += elided - break - else: - # Found single quote, check nearby text to eliminate digit separators. - # - # There is no special handling for floating point here, because - # the integer/fractional/exponent parts would all be parsed - # correctly as long as there are digits on both sides of the - # separator. So we are fine as long as we don't see something - # like "0.'3" (gcc 4.9.0 will not allow this literal). - if Search(r'\b(?:0[bBxX]?|[1-9])[0-9a-fA-F]*$', head): - match_literal = Match(r'^((?:\'?[0-9a-zA-Z_])*)(.*)$', "'" + tail) - collapsed += head + match_literal.group(1).replace("'", '') - elided = match_literal.group(2) - else: - second_quote = tail.find('\'') - if second_quote >= 0: - collapsed += head + "''" - elided = tail[second_quote + 1:] - else: - # Unmatched single quote - collapsed += elided - break - - return collapsed - - -def FindEndOfExpressionInLine(line, startpos, stack): - """Find the position just after the end of current parenthesized expression. - - Args: - line: a CleansedLines line. - startpos: start searching at this position. - stack: nesting stack at startpos. - - Returns: - On finding matching end: (index just after matching end, None) - On finding an unclosed expression: (-1, None) - Otherwise: (-1, new stack at end of this line) - """ - for i in xrange(startpos, len(line)): - char = line[i] - if char in '([{': - # Found start of parenthesized expression, push to expression stack - stack.append(char) - elif char == '<': - # Found potential start of template argument list - if i > 0 and line[i - 1] == '<': - # Left shift operator - if stack and stack[-1] == '<': - stack.pop() - if not stack: - return (-1, None) - elif i > 0 and Search(r'\boperator\s*$', line[0:i]): - # operator<, don't add to stack - continue - else: - # Tentative start of template argument list - stack.append('<') - elif char in ')]}': - # Found end of parenthesized expression. - # - # If we are currently expecting a matching '>', the pending '<' - # must have been an operator. Remove them from expression stack. - while stack and stack[-1] == '<': - stack.pop() - if not stack: - return (-1, None) - if ((stack[-1] == '(' and char == ')') or - (stack[-1] == '[' and char == ']') or - (stack[-1] == '{' and char == '}')): - stack.pop() - if not stack: - return (i + 1, None) - else: - # Mismatched parentheses - return (-1, None) - elif char == '>': - # Found potential end of template argument list. - - # Ignore "->" and operator functions - if (i > 0 and - (line[i - 1] == '-' or Search(r'\boperator\s*$', line[0:i - 1]))): - continue - - # Pop the stack if there is a matching '<'. Otherwise, ignore - # this '>' since it must be an operator. - if stack: - if stack[-1] == '<': - stack.pop() - if not stack: - return (i + 1, None) - elif char == ';': - # Found something that look like end of statements. If we are currently - # expecting a '>', the matching '<' must have been an operator, since - # template argument list should not contain statements. - while stack and stack[-1] == '<': - stack.pop() - if not stack: - return (-1, None) - - # Did not find end of expression or unbalanced parentheses on this line - return (-1, stack) - - -def CloseExpression(clean_lines, linenum, pos): - """If input points to ( or { or [ or <, finds the position that closes it. - - If lines[linenum][pos] points to a '(' or '{' or '[' or '<', finds the - linenum/pos that correspond to the closing of the expression. - - TODO(unknown): cpplint spends a fair bit of time matching parentheses. - Ideally we would want to index all opening and closing parentheses once - and have CloseExpression be just a simple lookup, but due to preprocessor - tricks, this is not so easy. - - Args: - clean_lines: A CleansedLines instance containing the file. - linenum: The number of the line to check. - pos: A position on the line. - - Returns: - A tuple (line, linenum, pos) pointer *past* the closing brace, or - (line, len(lines), -1) if we never find a close. Note we ignore - strings and comments when matching; and the line we return is the - 'cleansed' line at linenum. - """ - - line = clean_lines.elided[linenum] - if (line[pos] not in '({[<') or Match(r'<[<=]', line[pos:]): - return (line, clean_lines.NumLines(), -1) - - # Check first line - (end_pos, stack) = FindEndOfExpressionInLine(line, pos, []) - if end_pos > -1: - return (line, linenum, end_pos) - - # Continue scanning forward - while stack and linenum < clean_lines.NumLines() - 1: - linenum += 1 - line = clean_lines.elided[linenum] - (end_pos, stack) = FindEndOfExpressionInLine(line, 0, stack) - if end_pos > -1: - return (line, linenum, end_pos) - - # Did not find end of expression before end of file, give up - return (line, clean_lines.NumLines(), -1) - - -def FindStartOfExpressionInLine(line, endpos, stack): - """Find position at the matching start of current expression. - - This is almost the reverse of FindEndOfExpressionInLine, but note - that the input position and returned position differs by 1. - - Args: - line: a CleansedLines line. - endpos: start searching at this position. - stack: nesting stack at endpos. - - Returns: - On finding matching start: (index at matching start, None) - On finding an unclosed expression: (-1, None) - Otherwise: (-1, new stack at beginning of this line) - """ - i = endpos - while i >= 0: - char = line[i] - if char in ')]}': - # Found end of expression, push to expression stack - stack.append(char) - elif char == '>': - # Found potential end of template argument list. - # - # Ignore it if it's a "->" or ">=" or "operator>" - if (i > 0 and - (line[i - 1] == '-' or - Match(r'\s>=\s', line[i - 1:]) or - Search(r'\boperator\s*$', line[0:i]))): - i -= 1 - else: - stack.append('>') - elif char == '<': - # Found potential start of template argument list - if i > 0 and line[i - 1] == '<': - # Left shift operator - i -= 1 - else: - # If there is a matching '>', we can pop the expression stack. - # Otherwise, ignore this '<' since it must be an operator. - if stack and stack[-1] == '>': - stack.pop() - if not stack: - return (i, None) - elif char in '([{': - # Found start of expression. - # - # If there are any unmatched '>' on the stack, they must be - # operators. Remove those. - while stack and stack[-1] == '>': - stack.pop() - if not stack: - return (-1, None) - if ((char == '(' and stack[-1] == ')') or - (char == '[' and stack[-1] == ']') or - (char == '{' and stack[-1] == '}')): - stack.pop() - if not stack: - return (i, None) - else: - # Mismatched parentheses - return (-1, None) - elif char == ';': - # Found something that look like end of statements. If we are currently - # expecting a '<', the matching '>' must have been an operator, since - # template argument list should not contain statements. - while stack and stack[-1] == '>': - stack.pop() - if not stack: - return (-1, None) - - i -= 1 - - return (-1, stack) - - -def ReverseCloseExpression(clean_lines, linenum, pos): - """If input points to ) or } or ] or >, finds the position that opens it. - - If lines[linenum][pos] points to a ')' or '}' or ']' or '>', finds the - linenum/pos that correspond to the opening of the expression. - - Args: - clean_lines: A CleansedLines instance containing the file. - linenum: The number of the line to check. - pos: A position on the line. - - Returns: - A tuple (line, linenum, pos) pointer *at* the opening brace, or - (line, 0, -1) if we never find the matching opening brace. Note - we ignore strings and comments when matching; and the line we - return is the 'cleansed' line at linenum. - """ - line = clean_lines.elided[linenum] - if line[pos] not in ')}]>': - return (line, 0, -1) - - # Check last line - (start_pos, stack) = FindStartOfExpressionInLine(line, pos, []) - if start_pos > -1: - return (line, linenum, start_pos) - - # Continue scanning backward - while stack and linenum > 0: - linenum -= 1 - line = clean_lines.elided[linenum] - (start_pos, stack) = FindStartOfExpressionInLine(line, len(line) - 1, stack) - if start_pos > -1: - return (line, linenum, start_pos) - - # Did not find start of expression before beginning of file, give up - return (line, 0, -1) - - -def CheckForCopyright(filename, lines, error): - """Logs an error if no Copyright message appears at the top of the file.""" - - # We'll say it should occur by line 10. Don't forget there's a - # dummy line at the front. - for line in xrange(1, min(len(lines), 11)): - if re.search(r'Copyright', lines[line], re.I): break - else: # means no copyright line was found - error(filename, 0, 'legal/copyright', 5, - 'No copyright message found. ' - 'You should have a line: "Copyright [year] "') - - -def GetIndentLevel(line): - """Return the number of leading spaces in line. - - Args: - line: A string to check. - - Returns: - An integer count of leading spaces, possibly zero. - """ - indent = Match(r'^( *)\S', line) - if indent: - return len(indent.group(1)) - else: - return 0 - -def PathSplitToList(path): - """Returns the path split into a list by the separator. - - Args: - path: An absolute or relative path (e.g. '/a/b/c/' or '../a') - - Returns: - A list of path components (e.g. ['a', 'b', 'c]). - """ - lst = [] - while True: - (head, tail) = os.path.split(path) - if head == path: # absolute paths end - lst.append(head) - break - if tail == path: # relative paths end - lst.append(tail) - break - - path = head - lst.append(tail) - - lst.reverse() - return lst - -def GetHeaderGuardCPPVariable(filename): - """Returns the CPP variable that should be used as a header guard. - - Args: - filename: The name of a C++ header file. - - Returns: - The CPP variable that should be used as a header guard in the - named file. - - """ - - # Restores original filename in case that cpplint is invoked from Emacs's - # flymake. - filename = re.sub(r'_flymake\.h$', '.h', filename) - filename = re.sub(r'/\.flymake/([^/]*)$', r'/\1', filename) - # Replace 'c++' with 'cpp'. - filename = filename.replace('C++', 'cpp').replace('c++', 'cpp') - - fileinfo = FileInfo(filename) - file_path_from_root = fileinfo.RepositoryName() - - def FixupPathFromRoot(): - if _root_debug: - sys.stderr.write("\n_root fixup, _root = '%s', repository name = '%s'\n" - % (_root, fileinfo.RepositoryName())) - - # Process the file path with the --root flag if it was set. - if not _root: - if _root_debug: - sys.stderr.write("_root unspecified\n") - return file_path_from_root - - def StripListPrefix(lst, prefix): - # f(['x', 'y'], ['w, z']) -> None (not a valid prefix) - if lst[:len(prefix)] != prefix: - return None - # f(['a, 'b', 'c', 'd'], ['a', 'b']) -> ['c', 'd'] - return lst[(len(prefix)):] - - # root behavior: - # --root=subdir , lstrips subdir from the header guard - maybe_path = StripListPrefix(PathSplitToList(file_path_from_root), - PathSplitToList(_root)) - - if _root_debug: - sys.stderr.write(("_root lstrip (maybe_path=%s, file_path_from_root=%s," + - " _root=%s)\n") % (maybe_path, file_path_from_root, _root)) - - if maybe_path: - return os.path.join(*maybe_path) - - # --root=.. , will prepend the outer directory to the header guard - full_path = fileinfo.FullName() - root_abspath = os.path.abspath(_root) - - maybe_path = StripListPrefix(PathSplitToList(full_path), - PathSplitToList(root_abspath)) - - if _root_debug: - sys.stderr.write(("_root prepend (maybe_path=%s, full_path=%s, " + - "root_abspath=%s)\n") % (maybe_path, full_path, root_abspath)) - - if maybe_path: - return os.path.join(*maybe_path) - - if _root_debug: - sys.stderr.write("_root ignore, returning %s\n" % (file_path_from_root)) - - # --root=FAKE_DIR is ignored - return file_path_from_root - - file_path_from_root = FixupPathFromRoot() - return re.sub(r'[^a-zA-Z0-9]', '_', file_path_from_root).upper() + '_' - - -def CheckForHeaderGuard(filename, clean_lines, error): - """Checks that the file contains a header guard. - - Logs an error if no #ifndef header guard is present. For other - headers, checks that the full pathname is used. - - Args: - filename: The name of the C++ header file. - clean_lines: A CleansedLines instance containing the file. - error: The function to call with any errors found. - """ - - # Don't check for header guards if there are error suppression - # comments somewhere in this file. - # - # Because this is silencing a warning for a nonexistent line, we - # only support the very specific NOLINT(build/header_guard) syntax, - # and not the general NOLINT or NOLINT(*) syntax. - raw_lines = clean_lines.lines_without_raw_strings - for i in raw_lines: - if Search(r'//\s*NOLINT\(build/header_guard\)', i): - return - - # Allow pragma once instead of header guards - for i in raw_lines: - if Search(r'^\s*#pragma\s+once', i): - return - - cppvar = GetHeaderGuardCPPVariable(filename) - - ifndef = '' - ifndef_linenum = 0 - define = '' - endif = '' - endif_linenum = 0 - for linenum, line in enumerate(raw_lines): - linesplit = line.split() - if len(linesplit) >= 2: - # find the first occurrence of #ifndef and #define, save arg - if not ifndef and linesplit[0] == '#ifndef': - # set ifndef to the header guard presented on the #ifndef line. - ifndef = linesplit[1] - ifndef_linenum = linenum - if not define and linesplit[0] == '#define': - define = linesplit[1] - # find the last occurrence of #endif, save entire line - if line.startswith('#endif'): - endif = line - endif_linenum = linenum - - if not ifndef or not define or ifndef != define: - error(filename, 0, 'build/header_guard', 5, - 'No #ifndef header guard found, suggested CPP variable is: %s' % - cppvar) - return - - # The guard should be PATH_FILE_H_, but we also allow PATH_FILE_H__ - # for backward compatibility. - if ifndef != cppvar: - error_level = 0 - if ifndef != cppvar + '_': - error_level = 5 - - ParseNolintSuppressions(filename, raw_lines[ifndef_linenum], ifndef_linenum, - error) - error(filename, ifndef_linenum, 'build/header_guard', error_level, - '#ifndef header guard has wrong style, please use: %s' % cppvar) - - # Check for "//" comments on endif line. - ParseNolintSuppressions(filename, raw_lines[endif_linenum], endif_linenum, - error) - match = Match(r'#endif\s*//\s*' + cppvar + r'(_)?\b', endif) - if match: - if match.group(1) == '_': - # Issue low severity warning for deprecated double trailing underscore - error(filename, endif_linenum, 'build/header_guard', 0, - '#endif line should be "#endif // %s"' % cppvar) - return - - # Didn't find the corresponding "//" comment. If this file does not - # contain any "//" comments at all, it could be that the compiler - # only wants "/**/" comments, look for those instead. - no_single_line_comments = True - for i in xrange(1, len(raw_lines) - 1): - line = raw_lines[i] - if Match(r'^(?:(?:\'(?:\.|[^\'])*\')|(?:"(?:\.|[^"])*")|[^\'"])*//', line): - no_single_line_comments = False - break - - if no_single_line_comments: - match = Match(r'#endif\s*/\*\s*' + cppvar + r'(_)?\s*\*/', endif) - if match: - if match.group(1) == '_': - # Low severity warning for double trailing underscore - error(filename, endif_linenum, 'build/header_guard', 0, - '#endif line should be "#endif /* %s */"' % cppvar) - return - - # Didn't find anything - error(filename, endif_linenum, 'build/header_guard', 5, - '#endif line should be "#endif // %s"' % cppvar) - - -def CheckHeaderFileIncluded(filename, include_state, error): - """Logs an error if a source file does not include its header.""" - - # Do not check test files - fileinfo = FileInfo(filename) - if Search(_TEST_FILE_SUFFIX, fileinfo.BaseName()): - return - - for ext in GetHeaderExtensions(): - basefilename = filename[0:len(filename) - len(fileinfo.Extension())] - headerfile = basefilename + '.' + ext - if not os.path.exists(headerfile): - continue - headername = FileInfo(headerfile).RepositoryName() - first_include = None - for section_list in include_state.include_list: - for f in section_list: - if headername in f[0] or f[0] in headername: - return - if not first_include: - first_include = f[1] - - error(filename, first_include, 'build/include', 5, - '%s should include its header file %s' % (fileinfo.RepositoryName(), - headername)) - - -def CheckForBadCharacters(filename, lines, error): - """Logs an error for each line containing bad characters. - - Two kinds of bad characters: - - 1. Unicode replacement characters: These indicate that either the file - contained invalid UTF-8 (likely) or Unicode replacement characters (which - it shouldn't). Note that it's possible for this to throw off line - numbering if the invalid UTF-8 occurred adjacent to a newline. - - 2. NUL bytes. These are problematic for some tools. - - Args: - filename: The name of the current file. - lines: An array of strings, each representing a line of the file. - error: The function to call with any errors found. - """ - for linenum, line in enumerate(lines): - if unicode_escape_decode('\ufffd') in line: - error(filename, linenum, 'readability/utf8', 5, - 'Line contains invalid UTF-8 (or Unicode replacement character).') - if '\0' in line: - error(filename, linenum, 'readability/nul', 5, 'Line contains NUL byte.') - - -def CheckForNewlineAtEOF(filename, lines, error): - """Logs an error if there is no newline char at the end of the file. - - Args: - filename: The name of the current file. - lines: An array of strings, each representing a line of the file. - error: The function to call with any errors found. - """ - - # The array lines() was created by adding two newlines to the - # original file (go figure), then splitting on \n. - # To verify that the file ends in \n, we just have to make sure the - # last-but-two element of lines() exists and is empty. - if len(lines) < 3 or lines[-2]: - error(filename, len(lines) - 2, 'whitespace/ending_newline', 5, - 'Could not find a newline character at the end of the file.') - - -def CheckForMultilineCommentsAndStrings(filename, clean_lines, linenum, error): - """Logs an error if we see /* ... */ or "..." that extend past one line. - - /* ... */ comments are legit inside macros, for one line. - Otherwise, we prefer // comments, so it's ok to warn about the - other. Likewise, it's ok for strings to extend across multiple - lines, as long as a line continuation character (backslash) - terminates each line. Although not currently prohibited by the C++ - style guide, it's ugly and unnecessary. We don't do well with either - in this lint program, so we warn about both. - - Args: - filename: The name of the current file. - clean_lines: A CleansedLines instance containing the file. - linenum: The number of the line to check. - error: The function to call with any errors found. - """ - line = clean_lines.elided[linenum] - - # Remove all \\ (escaped backslashes) from the line. They are OK, and the - # second (escaped) slash may trigger later \" detection erroneously. - line = line.replace('\\\\', '') - - if line.count('/*') > line.count('*/'): - error(filename, linenum, 'readability/multiline_comment', 5, - 'Complex multi-line /*...*/-style comment found. ' - 'Lint may give bogus warnings. ' - 'Consider replacing these with //-style comments, ' - 'with #if 0...#endif, ' - 'or with more clearly structured multi-line comments.') - - if (line.count('"') - line.count('\\"')) % 2: - error(filename, linenum, 'readability/multiline_string', 5, - 'Multi-line string ("...") found. This lint script doesn\'t ' - 'do well with such strings, and may give bogus warnings. ' - 'Use C++11 raw strings or concatenation instead.') - - -# (non-threadsafe name, thread-safe alternative, validation pattern) -# -# The validation pattern is used to eliminate false positives such as: -# _rand(); // false positive due to substring match. -# ->rand(); // some member function rand(). -# ACMRandom rand(seed); // some variable named rand. -# ISAACRandom rand(); // another variable named rand. -# -# Basically we require the return value of these functions to be used -# in some expression context on the same line by matching on some -# operator before the function name. This eliminates constructors and -# member function calls. -_UNSAFE_FUNC_PREFIX = r'(?:[-+*/=%^&|(<]\s*|>\s+)' -_THREADING_LIST = ( - ('asctime(', 'asctime_r(', _UNSAFE_FUNC_PREFIX + r'asctime\([^)]+\)'), - ('ctime(', 'ctime_r(', _UNSAFE_FUNC_PREFIX + r'ctime\([^)]+\)'), - ('getgrgid(', 'getgrgid_r(', _UNSAFE_FUNC_PREFIX + r'getgrgid\([^)]+\)'), - ('getgrnam(', 'getgrnam_r(', _UNSAFE_FUNC_PREFIX + r'getgrnam\([^)]+\)'), - ('getlogin(', 'getlogin_r(', _UNSAFE_FUNC_PREFIX + r'getlogin\(\)'), - ('getpwnam(', 'getpwnam_r(', _UNSAFE_FUNC_PREFIX + r'getpwnam\([^)]+\)'), - ('getpwuid(', 'getpwuid_r(', _UNSAFE_FUNC_PREFIX + r'getpwuid\([^)]+\)'), - ('gmtime(', 'gmtime_r(', _UNSAFE_FUNC_PREFIX + r'gmtime\([^)]+\)'), - ('localtime(', 'localtime_r(', _UNSAFE_FUNC_PREFIX + r'localtime\([^)]+\)'), - ('rand(', 'rand_r(', _UNSAFE_FUNC_PREFIX + r'rand\(\)'), - ('strtok(', 'strtok_r(', - _UNSAFE_FUNC_PREFIX + r'strtok\([^)]+\)'), - ('ttyname(', 'ttyname_r(', _UNSAFE_FUNC_PREFIX + r'ttyname\([^)]+\)'), - ) - - -def CheckPosixThreading(filename, clean_lines, linenum, error): - """Checks for calls to thread-unsafe functions. - - Much code has been originally written without consideration of - multi-threading. Also, engineers are relying on their old experience; - they have learned posix before threading extensions were added. These - tests guide the engineers to use thread-safe functions (when using - posix directly). - - Args: - filename: The name of the current file. - clean_lines: A CleansedLines instance containing the file. - linenum: The number of the line to check. - error: The function to call with any errors found. - """ - line = clean_lines.elided[linenum] - for single_thread_func, multithread_safe_func, pattern in _THREADING_LIST: - # Additional pattern matching check to confirm that this is the - # function we are looking for - if Search(pattern, line): - error(filename, linenum, 'runtime/threadsafe_fn', 2, - 'Consider using ' + multithread_safe_func + - '...) instead of ' + single_thread_func + - '...) for improved thread safety.') - - -def CheckVlogArguments(filename, clean_lines, linenum, error): - """Checks that VLOG() is only used for defining a logging level. - - For example, VLOG(2) is correct. VLOG(INFO), VLOG(WARNING), VLOG(ERROR), and - VLOG(FATAL) are not. - - Args: - filename: The name of the current file. - clean_lines: A CleansedLines instance containing the file. - linenum: The number of the line to check. - error: The function to call with any errors found. - """ - line = clean_lines.elided[linenum] - if Search(r'\bVLOG\((INFO|ERROR|WARNING|DFATAL|FATAL)\)', line): - error(filename, linenum, 'runtime/vlog', 5, - 'VLOG() should be used with numeric verbosity level. ' - 'Use LOG() if you want symbolic severity levels.') - -# Matches invalid increment: *count++, which moves pointer instead of -# incrementing a value. -_RE_PATTERN_INVALID_INCREMENT = re.compile( - r'^\s*\*\w+(\+\+|--);') - - -def CheckInvalidIncrement(filename, clean_lines, linenum, error): - """Checks for invalid increment *count++. - - For example following function: - void increment_counter(int* count) { - *count++; - } - is invalid, because it effectively does count++, moving pointer, and should - be replaced with ++*count, (*count)++ or *count += 1. - - Args: - filename: The name of the current file. - clean_lines: A CleansedLines instance containing the file. - linenum: The number of the line to check. - error: The function to call with any errors found. - """ - line = clean_lines.elided[linenum] - if _RE_PATTERN_INVALID_INCREMENT.match(line): - error(filename, linenum, 'runtime/invalid_increment', 5, - 'Changing pointer instead of value (or unused value of operator*).') - - -def IsMacroDefinition(clean_lines, linenum): - if Search(r'^#define', clean_lines[linenum]): - return True - - if linenum > 0 and Search(r'\\$', clean_lines[linenum - 1]): - return True - - return False - - -def IsForwardClassDeclaration(clean_lines, linenum): - return Match(r'^\s*(\btemplate\b)*.*class\s+\w+;\s*$', clean_lines[linenum]) - - -class _BlockInfo(object): - """Stores information about a generic block of code.""" - - def __init__(self, linenum, seen_open_brace): - self.starting_linenum = linenum - self.seen_open_brace = seen_open_brace - self.open_parentheses = 0 - self.inline_asm = _NO_ASM - self.check_namespace_indentation = False - - def CheckBegin(self, filename, clean_lines, linenum, error): - """Run checks that applies to text up to the opening brace. - - This is mostly for checking the text after the class identifier - and the "{", usually where the base class is specified. For other - blocks, there isn't much to check, so we always pass. - - Args: - filename: The name of the current file. - clean_lines: A CleansedLines instance containing the file. - linenum: The number of the line to check. - error: The function to call with any errors found. - """ - pass - - def CheckEnd(self, filename, clean_lines, linenum, error): - """Run checks that applies to text after the closing brace. - - This is mostly used for checking end of namespace comments. - - Args: - filename: The name of the current file. - clean_lines: A CleansedLines instance containing the file. - linenum: The number of the line to check. - error: The function to call with any errors found. - """ - pass - - def IsBlockInfo(self): - """Returns true if this block is a _BlockInfo. - - This is convenient for verifying that an object is an instance of - a _BlockInfo, but not an instance of any of the derived classes. - - Returns: - True for this class, False for derived classes. - """ - return self.__class__ == _BlockInfo - - -class _ExternCInfo(_BlockInfo): - """Stores information about an 'extern "C"' block.""" - - def __init__(self, linenum): - _BlockInfo.__init__(self, linenum, True) - - -class _ClassInfo(_BlockInfo): - """Stores information about a class.""" - - def __init__(self, name, class_or_struct, clean_lines, linenum): - _BlockInfo.__init__(self, linenum, False) - self.name = name - self.is_derived = False - self.check_namespace_indentation = True - if class_or_struct == 'struct': - self.access = 'public' - self.is_struct = True - else: - self.access = 'private' - self.is_struct = False - - # Remember initial indentation level for this class. Using raw_lines here - # instead of elided to account for leading comments. - self.class_indent = GetIndentLevel(clean_lines.raw_lines[linenum]) - - # Try to find the end of the class. This will be confused by things like: - # class A { - # } *x = { ... - # - # But it's still good enough for CheckSectionSpacing. - self.last_line = 0 - depth = 0 - for i in range(linenum, clean_lines.NumLines()): - line = clean_lines.elided[i] - depth += line.count('{') - line.count('}') - if not depth: - self.last_line = i - break - - def CheckBegin(self, filename, clean_lines, linenum, error): - # Look for a bare ':' - if Search('(^|[^:]):($|[^:])', clean_lines.elided[linenum]): - self.is_derived = True - - def CheckEnd(self, filename, clean_lines, linenum, error): - # If there is a DISALLOW macro, it should appear near the end of - # the class. - seen_last_thing_in_class = False - for i in xrange(linenum - 1, self.starting_linenum, -1): - match = Search( - r'\b(DISALLOW_COPY_AND_ASSIGN|DISALLOW_IMPLICIT_CONSTRUCTORS)\(' + - self.name + r'\)', - clean_lines.elided[i]) - if match: - if seen_last_thing_in_class: - error(filename, i, 'readability/constructors', 3, - match.group(1) + ' should be the last thing in the class') - break - - if not Match(r'^\s*$', clean_lines.elided[i]): - seen_last_thing_in_class = True - - # Check that closing brace is aligned with beginning of the class. - # Only do this if the closing brace is indented by only whitespaces. - # This means we will not check single-line class definitions. - indent = Match(r'^( *)\}', clean_lines.elided[linenum]) - if indent and len(indent.group(1)) != self.class_indent: - if self.is_struct: - parent = 'struct ' + self.name - else: - parent = 'class ' + self.name - error(filename, linenum, 'whitespace/indent', 3, - 'Closing brace should be aligned with beginning of %s' % parent) - - -class _NamespaceInfo(_BlockInfo): - """Stores information about a namespace.""" - - def __init__(self, name, linenum): - _BlockInfo.__init__(self, linenum, False) - self.name = name or '' - self.check_namespace_indentation = True - - def CheckEnd(self, filename, clean_lines, linenum, error): - """Check end of namespace comments.""" - line = clean_lines.raw_lines[linenum] - - # Check how many lines is enclosed in this namespace. Don't issue - # warning for missing namespace comments if there aren't enough - # lines. However, do apply checks if there is already an end of - # namespace comment and it's incorrect. - # - # TODO(unknown): We always want to check end of namespace comments - # if a namespace is large, but sometimes we also want to apply the - # check if a short namespace contained nontrivial things (something - # other than forward declarations). There is currently no logic on - # deciding what these nontrivial things are, so this check is - # triggered by namespace size only, which works most of the time. - if (linenum - self.starting_linenum < 10 - and not Match(r'^\s*};*\s*(//|/\*).*\bnamespace\b', line)): - return - - # Look for matching comment at end of namespace. - # - # Note that we accept C style "/* */" comments for terminating - # namespaces, so that code that terminate namespaces inside - # preprocessor macros can be cpplint clean. - # - # We also accept stuff like "// end of namespace ." with the - # period at the end. - # - # Besides these, we don't accept anything else, otherwise we might - # get false negatives when existing comment is a substring of the - # expected namespace. - if self.name: - # Named namespace - if not Match((r'^\s*};*\s*(//|/\*).*\bnamespace\s+' + - re.escape(self.name) + r'[\*/\.\\\s]*$'), - line): - error(filename, linenum, 'readability/namespace', 5, - 'Namespace should be terminated with "// namespace %s"' % - self.name) - else: - # Anonymous namespace - if not Match(r'^\s*};*\s*(//|/\*).*\bnamespace[\*/\.\\\s]*$', line): - # If "// namespace anonymous" or "// anonymous namespace (more text)", - # mention "// anonymous namespace" as an acceptable form - if Match(r'^\s*}.*\b(namespace anonymous|anonymous namespace)\b', line): - error(filename, linenum, 'readability/namespace', 5, - 'Anonymous namespace should be terminated with "// namespace"' - ' or "// anonymous namespace"') - else: - error(filename, linenum, 'readability/namespace', 5, - 'Anonymous namespace should be terminated with "// namespace"') - - -class _PreprocessorInfo(object): - """Stores checkpoints of nesting stacks when #if/#else is seen.""" - - def __init__(self, stack_before_if): - # The entire nesting stack before #if - self.stack_before_if = stack_before_if - - # The entire nesting stack up to #else - self.stack_before_else = [] - - # Whether we have already seen #else or #elif - self.seen_else = False - - -class NestingState(object): - """Holds states related to parsing braces.""" - - def __init__(self): - # Stack for tracking all braces. An object is pushed whenever we - # see a "{", and popped when we see a "}". Only 3 types of - # objects are possible: - # - _ClassInfo: a class or struct. - # - _NamespaceInfo: a namespace. - # - _BlockInfo: some other type of block. - self.stack = [] - - # Top of the previous stack before each Update(). - # - # Because the nesting_stack is updated at the end of each line, we - # had to do some convoluted checks to find out what is the current - # scope at the beginning of the line. This check is simplified by - # saving the previous top of nesting stack. - # - # We could save the full stack, but we only need the top. Copying - # the full nesting stack would slow down cpplint by ~10%. - self.previous_stack_top = [] - - # Stack of _PreprocessorInfo objects. - self.pp_stack = [] - - def SeenOpenBrace(self): - """Check if we have seen the opening brace for the innermost block. - - Returns: - True if we have seen the opening brace, False if the innermost - block is still expecting an opening brace. - """ - return (not self.stack) or self.stack[-1].seen_open_brace - - def InNamespaceBody(self): - """Check if we are currently one level inside a namespace body. - - Returns: - True if top of the stack is a namespace block, False otherwise. - """ - return self.stack and isinstance(self.stack[-1], _NamespaceInfo) - - def InExternC(self): - """Check if we are currently one level inside an 'extern "C"' block. - - Returns: - True if top of the stack is an extern block, False otherwise. - """ - return self.stack and isinstance(self.stack[-1], _ExternCInfo) - - def InClassDeclaration(self): - """Check if we are currently one level inside a class or struct declaration. - - Returns: - True if top of the stack is a class/struct, False otherwise. - """ - return self.stack and isinstance(self.stack[-1], _ClassInfo) - - def InAsmBlock(self): - """Check if we are currently one level inside an inline ASM block. - - Returns: - True if the top of the stack is a block containing inline ASM. - """ - return self.stack and self.stack[-1].inline_asm != _NO_ASM - - def InTemplateArgumentList(self, clean_lines, linenum, pos): - """Check if current position is inside template argument list. - - Args: - clean_lines: A CleansedLines instance containing the file. - linenum: The number of the line to check. - pos: position just after the suspected template argument. - Returns: - True if (linenum, pos) is inside template arguments. - """ - while linenum < clean_lines.NumLines(): - # Find the earliest character that might indicate a template argument - line = clean_lines.elided[linenum] - match = Match(r'^[^{};=\[\]\.<>]*(.)', line[pos:]) - if not match: - linenum += 1 - pos = 0 - continue - token = match.group(1) - pos += len(match.group(0)) - - # These things do not look like template argument list: - # class Suspect { - # class Suspect x; } - if token in ('{', '}', ';'): return False - - # These things look like template argument list: - # template - # template - # template - # template - if token in ('>', '=', '[', ']', '.'): return True - - # Check if token is an unmatched '<'. - # If not, move on to the next character. - if token != '<': - pos += 1 - if pos >= len(line): - linenum += 1 - pos = 0 - continue - - # We can't be sure if we just find a single '<', and need to - # find the matching '>'. - (_, end_line, end_pos) = CloseExpression(clean_lines, linenum, pos - 1) - if end_pos < 0: - # Not sure if template argument list or syntax error in file - return False - linenum = end_line - pos = end_pos - return False - - def UpdatePreprocessor(self, line): - """Update preprocessor stack. - - We need to handle preprocessors due to classes like this: - #ifdef SWIG - struct ResultDetailsPageElementExtensionPoint { - #else - struct ResultDetailsPageElementExtensionPoint : public Extension { - #endif - - We make the following assumptions (good enough for most files): - - Preprocessor condition evaluates to true from #if up to first - #else/#elif/#endif. - - - Preprocessor condition evaluates to false from #else/#elif up - to #endif. We still perform lint checks on these lines, but - these do not affect nesting stack. - - Args: - line: current line to check. - """ - if Match(r'^\s*#\s*(if|ifdef|ifndef)\b', line): - # Beginning of #if block, save the nesting stack here. The saved - # stack will allow us to restore the parsing state in the #else case. - self.pp_stack.append(_PreprocessorInfo(copy.deepcopy(self.stack))) - elif Match(r'^\s*#\s*(else|elif)\b', line): - # Beginning of #else block - if self.pp_stack: - if not self.pp_stack[-1].seen_else: - # This is the first #else or #elif block. Remember the - # whole nesting stack up to this point. This is what we - # keep after the #endif. - self.pp_stack[-1].seen_else = True - self.pp_stack[-1].stack_before_else = copy.deepcopy(self.stack) - - # Restore the stack to how it was before the #if - self.stack = copy.deepcopy(self.pp_stack[-1].stack_before_if) - else: - # TODO(unknown): unexpected #else, issue warning? - pass - elif Match(r'^\s*#\s*endif\b', line): - # End of #if or #else blocks. - if self.pp_stack: - # If we saw an #else, we will need to restore the nesting - # stack to its former state before the #else, otherwise we - # will just continue from where we left off. - if self.pp_stack[-1].seen_else: - # Here we can just use a shallow copy since we are the last - # reference to it. - self.stack = self.pp_stack[-1].stack_before_else - # Drop the corresponding #if - self.pp_stack.pop() - else: - # TODO(unknown): unexpected #endif, issue warning? - pass - - # TODO(unknown): Update() is too long, but we will refactor later. - def Update(self, filename, clean_lines, linenum, error): - """Update nesting state with current line. - - Args: - filename: The name of the current file. - clean_lines: A CleansedLines instance containing the file. - linenum: The number of the line to check. - error: The function to call with any errors found. - """ - line = clean_lines.elided[linenum] - - # Remember top of the previous nesting stack. - # - # The stack is always pushed/popped and not modified in place, so - # we can just do a shallow copy instead of copy.deepcopy. Using - # deepcopy would slow down cpplint by ~28%. - if self.stack: - self.previous_stack_top = self.stack[-1] - else: - self.previous_stack_top = None - - # Update pp_stack - self.UpdatePreprocessor(line) - - # Count parentheses. This is to avoid adding struct arguments to - # the nesting stack. - if self.stack: - inner_block = self.stack[-1] - depth_change = line.count('(') - line.count(')') - inner_block.open_parentheses += depth_change - - # Also check if we are starting or ending an inline assembly block. - if inner_block.inline_asm in (_NO_ASM, _END_ASM): - if (depth_change != 0 and - inner_block.open_parentheses == 1 and - _MATCH_ASM.match(line)): - # Enter assembly block - inner_block.inline_asm = _INSIDE_ASM - else: - # Not entering assembly block. If previous line was _END_ASM, - # we will now shift to _NO_ASM state. - inner_block.inline_asm = _NO_ASM - elif (inner_block.inline_asm == _INSIDE_ASM and - inner_block.open_parentheses == 0): - # Exit assembly block - inner_block.inline_asm = _END_ASM - - # Consume namespace declaration at the beginning of the line. Do - # this in a loop so that we catch same line declarations like this: - # namespace proto2 { namespace bridge { class MessageSet; } } - while True: - # Match start of namespace. The "\b\s*" below catches namespace - # declarations even if it weren't followed by a whitespace, this - # is so that we don't confuse our namespace checker. The - # missing spaces will be flagged by CheckSpacing. - namespace_decl_match = Match(r'^\s*namespace\b\s*([:\w]+)?(.*)$', line) - if not namespace_decl_match: - break - - new_namespace = _NamespaceInfo(namespace_decl_match.group(1), linenum) - self.stack.append(new_namespace) - - line = namespace_decl_match.group(2) - if line.find('{') != -1: - new_namespace.seen_open_brace = True - line = line[line.find('{') + 1:] - - # Look for a class declaration in whatever is left of the line - # after parsing namespaces. The regexp accounts for decorated classes - # such as in: - # class LOCKABLE API Object { - # }; - class_decl_match = Match( - r'^(\s*(?:template\s*<[\w\s<>,:=]*>\s*)?' - r'(class|struct)\s+(?:[A-Z_]+\s+)*(\w+(?:::\w+)*))' - r'(.*)$', line) - if (class_decl_match and - (not self.stack or self.stack[-1].open_parentheses == 0)): - # We do not want to accept classes that are actually template arguments: - # template , - # template class Ignore3> - # void Function() {}; - # - # To avoid template argument cases, we scan forward and look for - # an unmatched '>'. If we see one, assume we are inside a - # template argument list. - end_declaration = len(class_decl_match.group(1)) - if not self.InTemplateArgumentList(clean_lines, linenum, end_declaration): - self.stack.append(_ClassInfo( - class_decl_match.group(3), class_decl_match.group(2), - clean_lines, linenum)) - line = class_decl_match.group(4) - - # If we have not yet seen the opening brace for the innermost block, - # run checks here. - if not self.SeenOpenBrace(): - self.stack[-1].CheckBegin(filename, clean_lines, linenum, error) - - # Update access control if we are inside a class/struct - if self.stack and isinstance(self.stack[-1], _ClassInfo): - classinfo = self.stack[-1] - access_match = Match( - r'^(.*)\b(public|private|protected|signals)(\s+(?:slots\s*)?)?' - r':(?:[^:]|$)', - line) - if access_match: - classinfo.access = access_match.group(2) - - # Check that access keywords are indented +1 space. Skip this - # check if the keywords are not preceded by whitespaces. - indent = access_match.group(1) - if (len(indent) != classinfo.class_indent + 1 and - Match(r'^\s*$', indent)): - if classinfo.is_struct: - parent = 'struct ' + classinfo.name - else: - parent = 'class ' + classinfo.name - slots = '' - if access_match.group(3): - slots = access_match.group(3) - error(filename, linenum, 'whitespace/indent', 3, - '%s%s: should be indented +1 space inside %s' % ( - access_match.group(2), slots, parent)) - - # Consume braces or semicolons from what's left of the line - while True: - # Match first brace, semicolon, or closed parenthesis. - matched = Match(r'^[^{;)}]*([{;)}])(.*)$', line) - if not matched: - break - - token = matched.group(1) - if token == '{': - # If namespace or class hasn't seen a opening brace yet, mark - # namespace/class head as complete. Push a new block onto the - # stack otherwise. - if not self.SeenOpenBrace(): - self.stack[-1].seen_open_brace = True - elif Match(r'^extern\s*"[^"]*"\s*\{', line): - self.stack.append(_ExternCInfo(linenum)) - else: - self.stack.append(_BlockInfo(linenum, True)) - if _MATCH_ASM.match(line): - self.stack[-1].inline_asm = _BLOCK_ASM - - elif token == ';' or token == ')': - # If we haven't seen an opening brace yet, but we already saw - # a semicolon, this is probably a forward declaration. Pop - # the stack for these. - # - # Similarly, if we haven't seen an opening brace yet, but we - # already saw a closing parenthesis, then these are probably - # function arguments with extra "class" or "struct" keywords. - # Also pop these stack for these. - if not self.SeenOpenBrace(): - self.stack.pop() - else: # token == '}' - # Perform end of block checks and pop the stack. - if self.stack: - self.stack[-1].CheckEnd(filename, clean_lines, linenum, error) - self.stack.pop() - line = matched.group(2) - - def InnermostClass(self): - """Get class info on the top of the stack. - - Returns: - A _ClassInfo object if we are inside a class, or None otherwise. - """ - for i in range(len(self.stack), 0, -1): - classinfo = self.stack[i - 1] - if isinstance(classinfo, _ClassInfo): - return classinfo - return None - - def CheckCompletedBlocks(self, filename, error): - """Checks that all classes and namespaces have been completely parsed. - - Call this when all lines in a file have been processed. - Args: - filename: The name of the current file. - error: The function to call with any errors found. - """ - # Note: This test can result in false positives if #ifdef constructs - # get in the way of brace matching. See the testBuildClass test in - # cpplint_unittest.py for an example of this. - for obj in self.stack: - if isinstance(obj, _ClassInfo): - error(filename, obj.starting_linenum, 'build/class', 5, - 'Failed to find complete declaration of class %s' % - obj.name) - elif isinstance(obj, _NamespaceInfo): - error(filename, obj.starting_linenum, 'build/namespaces', 5, - 'Failed to find complete declaration of namespace %s' % - obj.name) - - -def CheckForNonStandardConstructs(filename, clean_lines, linenum, - nesting_state, error): - r"""Logs an error if we see certain non-ANSI constructs ignored by gcc-2. - - Complain about several constructs which gcc-2 accepts, but which are - not standard C++. Warning about these in lint is one way to ease the - transition to new compilers. - - put storage class first (e.g. "static const" instead of "const static"). - - "%lld" instead of %qd" in printf-type functions. - - "%1$d" is non-standard in printf-type functions. - - "\%" is an undefined character escape sequence. - - text after #endif is not allowed. - - invalid inner-style forward declaration. - - >? and ?= and )\?=?\s*(\w+|[+-]?\d+)(\.\d*)?', - line): - error(filename, linenum, 'build/deprecated', 3, - '>? and ))?' - # r'\s*const\s*' + type_name + '\s*&\s*\w+\s*;' - error(filename, linenum, 'runtime/member_string_references', 2, - 'const string& members are dangerous. It is much better to use ' - 'alternatives, such as pointers or simple constants.') - - # Everything else in this function operates on class declarations. - # Return early if the top of the nesting stack is not a class, or if - # the class head is not completed yet. - classinfo = nesting_state.InnermostClass() - if not classinfo or not classinfo.seen_open_brace: - return - - # The class may have been declared with namespace or classname qualifiers. - # The constructor and destructor will not have those qualifiers. - base_classname = classinfo.name.split('::')[-1] - - # Look for single-argument constructors that aren't marked explicit. - # Technically a valid construct, but against style. - explicit_constructor_match = Match( - r'\s+(?:(?:inline|constexpr)\s+)*(explicit\s+)?' - r'(?:(?:inline|constexpr)\s+)*%s\s*' - r'\(((?:[^()]|\([^()]*\))*)\)' - % re.escape(base_classname), - line) - - if explicit_constructor_match: - is_marked_explicit = explicit_constructor_match.group(1) - - if not explicit_constructor_match.group(2): - constructor_args = [] - else: - constructor_args = explicit_constructor_match.group(2).split(',') - - # collapse arguments so that commas in template parameter lists and function - # argument parameter lists don't split arguments in two - i = 0 - while i < len(constructor_args): - constructor_arg = constructor_args[i] - while (constructor_arg.count('<') > constructor_arg.count('>') or - constructor_arg.count('(') > constructor_arg.count(')')): - constructor_arg += ',' + constructor_args[i + 1] - del constructor_args[i + 1] - constructor_args[i] = constructor_arg - i += 1 - - variadic_args = [arg for arg in constructor_args if '&&...' in arg] - defaulted_args = [arg for arg in constructor_args if '=' in arg] - noarg_constructor = (not constructor_args or # empty arg list - # 'void' arg specifier - (len(constructor_args) == 1 and - constructor_args[0].strip() == 'void')) - onearg_constructor = ((len(constructor_args) == 1 and # exactly one arg - not noarg_constructor) or - # all but at most one arg defaulted - (len(constructor_args) >= 1 and - not noarg_constructor and - len(defaulted_args) >= len(constructor_args) - 1) or - # variadic arguments with zero or one argument - (len(constructor_args) <= 2 and - len(variadic_args) >= 1)) - initializer_list_constructor = bool( - onearg_constructor and - Search(r'\bstd\s*::\s*initializer_list\b', constructor_args[0])) - copy_constructor = bool( - onearg_constructor and - Match(r'(const\s+)?%s(\s*<[^>]*>)?(\s+const)?\s*(?:<\w+>\s*)?&' - % re.escape(base_classname), constructor_args[0].strip())) - - if (not is_marked_explicit and - onearg_constructor and - not initializer_list_constructor and - not copy_constructor): - if defaulted_args or variadic_args: - error(filename, linenum, 'runtime/explicit', 5, - 'Constructors callable with one argument ' - 'should be marked explicit.') - else: - error(filename, linenum, 'runtime/explicit', 5, - 'Single-parameter constructors should be marked explicit.') - elif is_marked_explicit and not onearg_constructor: - if noarg_constructor: - error(filename, linenum, 'runtime/explicit', 5, - 'Zero-parameter constructors should not be marked explicit.') - - -def CheckSpacingForFunctionCall(filename, clean_lines, linenum, error): - """Checks for the correctness of various spacing around function calls. - - Args: - filename: The name of the current file. - clean_lines: A CleansedLines instance containing the file. - linenum: The number of the line to check. - error: The function to call with any errors found. - """ - line = clean_lines.elided[linenum] - - # Since function calls often occur inside if/for/while/switch - # expressions - which have their own, more liberal conventions - we - # first see if we should be looking inside such an expression for a - # function call, to which we can apply more strict standards. - fncall = line # if there's no control flow construct, look at whole line - for pattern in (r'\bif\s*\((.*)\)\s*{', - r'\bfor\s*\((.*)\)\s*{', - r'\bwhile\s*\((.*)\)\s*[{;]', - r'\bswitch\s*\((.*)\)\s*{'): - match = Search(pattern, line) - if match: - fncall = match.group(1) # look inside the parens for function calls - break - - # Except in if/for/while/switch, there should never be space - # immediately inside parens (eg "f( 3, 4 )"). We make an exception - # for nested parens ( (a+b) + c ). Likewise, there should never be - # a space before a ( when it's a function argument. I assume it's a - # function argument when the char before the whitespace is legal in - # a function name (alnum + _) and we're not starting a macro. Also ignore - # pointers and references to arrays and functions coz they're too tricky: - # we use a very simple way to recognize these: - # " (something)(maybe-something)" or - # " (something)(maybe-something," or - # " (something)[something]" - # Note that we assume the contents of [] to be short enough that - # they'll never need to wrap. - if ( # Ignore control structures. - not Search(r'\b(if|for|while|switch|return|new|delete|catch|sizeof)\b', - fncall) and - # Ignore pointers/references to functions. - not Search(r' \([^)]+\)\([^)]*(\)|,$)', fncall) and - # Ignore pointers/references to arrays. - not Search(r' \([^)]+\)\[[^\]]+\]', fncall)): - if Search(r'\w\s*\(\s(?!\s*\\$)', fncall): # a ( used for a fn call - error(filename, linenum, 'whitespace/parens', 4, - 'Extra space after ( in function call') - elif Search(r'\(\s+(?!(\s*\\)|\()', fncall): - error(filename, linenum, 'whitespace/parens', 2, - 'Extra space after (') - if (Search(r'\w\s+\(', fncall) and - not Search(r'_{0,2}asm_{0,2}\s+_{0,2}volatile_{0,2}\s+\(', fncall) and - not Search(r'#\s*define|typedef|using\s+\w+\s*=', fncall) and - not Search(r'\w\s+\((\w+::)*\*\w+\)\(', fncall) and - not Search(r'\bcase\s+\(', fncall)): - # TODO(unknown): Space after an operator function seem to be a common - # error, silence those for now by restricting them to highest verbosity. - if Search(r'\boperator_*\b', line): - error(filename, linenum, 'whitespace/parens', 0, - 'Extra space before ( in function call') - else: - error(filename, linenum, 'whitespace/parens', 4, - 'Extra space before ( in function call') - # If the ) is followed only by a newline or a { + newline, assume it's - # part of a control statement (if/while/etc), and don't complain - if Search(r'[^)]\s+\)\s*[^{\s]', fncall): - # If the closing parenthesis is preceded by only whitespaces, - # try to give a more descriptive error message. - if Search(r'^\s+\)', fncall): - error(filename, linenum, 'whitespace/parens', 2, - 'Closing ) should be moved to the previous line') - else: - error(filename, linenum, 'whitespace/parens', 2, - 'Extra space before )') - - -def IsBlankLine(line): - """Returns true if the given line is blank. - - We consider a line to be blank if the line is empty or consists of - only white spaces. - - Args: - line: A line of a string. - - Returns: - True, if the given line is blank. - """ - return not line or line.isspace() - - -def CheckForNamespaceIndentation(filename, nesting_state, clean_lines, line, - error): - is_namespace_indent_item = ( - len(nesting_state.stack) > 1 and - nesting_state.stack[-1].check_namespace_indentation and - isinstance(nesting_state.previous_stack_top, _NamespaceInfo) and - nesting_state.previous_stack_top == nesting_state.stack[-2]) - - if ShouldCheckNamespaceIndentation(nesting_state, is_namespace_indent_item, - clean_lines.elided, line): - CheckItemIndentationInNamespace(filename, clean_lines.elided, - line, error) - - -def CheckForFunctionLengths(filename, clean_lines, linenum, - function_state, error): - """Reports for long function bodies. - - For an overview why this is done, see: - https://google-styleguide.googlecode.com/svn/trunk/cppguide.xml#Write_Short_Functions - - Uses a simplistic algorithm assuming other style guidelines - (especially spacing) are followed. - Only checks unindented functions, so class members are unchecked. - Trivial bodies are unchecked, so constructors with huge initializer lists - may be missed. - Blank/comment lines are not counted so as to avoid encouraging the removal - of vertical space and comments just to get through a lint check. - NOLINT *on the last line of a function* disables this check. - - Args: - filename: The name of the current file. - clean_lines: A CleansedLines instance containing the file. - linenum: The number of the line to check. - function_state: Current function name and lines in body so far. - error: The function to call with any errors found. - """ - lines = clean_lines.lines - line = lines[linenum] - joined_line = '' - - starting_func = False - regexp = r'(\w(\w|::|\*|\&|\s)*)\(' # decls * & space::name( ... - match_result = Match(regexp, line) - if match_result: - # If the name is all caps and underscores, figure it's a macro and - # ignore it, unless it's TEST or TEST_F. - function_name = match_result.group(1).split()[-1] - if function_name == 'TEST' or function_name == 'TEST_F' or ( - not Match(r'[A-Z_]+$', function_name)): - starting_func = True - - if starting_func: - body_found = False - for start_linenum in xrange(linenum, clean_lines.NumLines()): - start_line = lines[start_linenum] - joined_line += ' ' + start_line.lstrip() - if Search(r'(;|})', start_line): # Declarations and trivial functions - body_found = True - break # ... ignore - if Search(r'{', start_line): - body_found = True - function = Search(r'((\w|:)*)\(', line).group(1) - if Match(r'TEST', function): # Handle TEST... macros - parameter_regexp = Search(r'(\(.*\))', joined_line) - if parameter_regexp: # Ignore bad syntax - function += parameter_regexp.group(1) - else: - function += '()' - function_state.Begin(function) - break - if not body_found: - # No body for the function (or evidence of a non-function) was found. - error(filename, linenum, 'readability/fn_size', 5, - 'Lint failed to find start of function body.') - elif Match(r'^\}\s*$', line): # function end - function_state.Check(error, filename, linenum) - function_state.End() - elif not Match(r'^\s*$', line): - function_state.Count() # Count non-blank/non-comment lines. - - -_RE_PATTERN_TODO = re.compile(r'^//(\s*)TODO(\(.+?\))?:?(\s|$)?') - - -def CheckComment(line, filename, linenum, next_line_start, error): - """Checks for common mistakes in comments. - - Args: - line: The line in question. - filename: The name of the current file. - linenum: The number of the line to check. - next_line_start: The first non-whitespace column of the next line. - error: The function to call with any errors found. - """ - commentpos = line.find('//') - if commentpos != -1: - # Check if the // may be in quotes. If so, ignore it - if re.sub(r'\\.', '', line[0:commentpos]).count('"') % 2 == 0: - # Allow one space for new scopes, two spaces otherwise: - if (not (Match(r'^.*{ *//', line) and next_line_start == commentpos) and - ((commentpos >= 1 and - line[commentpos-1] not in string.whitespace) or - (commentpos >= 2 and - line[commentpos-2] not in string.whitespace))): - error(filename, linenum, 'whitespace/comments', 2, - 'At least two spaces is best between code and comments') - - # Checks for common mistakes in TODO comments. - comment = line[commentpos:] - match = _RE_PATTERN_TODO.match(comment) - if match: - # One whitespace is correct; zero whitespace is handled elsewhere. - leading_whitespace = match.group(1) - if len(leading_whitespace) > 1: - error(filename, linenum, 'whitespace/todo', 2, - 'Too many spaces before TODO') - - username = match.group(2) - if not username: - error(filename, linenum, 'readability/todo', 2, - 'Missing username in TODO; it should look like ' - '"// TODO(my_username): Stuff."') - - middle_whitespace = match.group(3) - # Comparisons made explicit for correctness -- pylint: disable=g-explicit-bool-comparison - if middle_whitespace != ' ' and middle_whitespace != '': - error(filename, linenum, 'whitespace/todo', 2, - 'TODO(my_username) should be followed by a space') - - # If the comment contains an alphanumeric character, there - # should be a space somewhere between it and the // unless - # it's a /// or //! Doxygen comment. - if (Match(r'//[^ ]*\w', comment) and - not Match(r'(///|//\!)(\s+|$)', comment)): - error(filename, linenum, 'whitespace/comments', 4, - 'Should have a space between // and comment') - - -def CheckSpacing(filename, clean_lines, linenum, nesting_state, error): - """Checks for the correctness of various spacing issues in the code. - - Things we check for: spaces around operators, spaces after - if/for/while/switch, no spaces around parens in function calls, two - spaces between code and comment, don't start a block with a blank - line, don't end a function with a blank line, don't add a blank line - after public/protected/private, don't have too many blank lines in a row. - - Args: - filename: The name of the current file. - clean_lines: A CleansedLines instance containing the file. - linenum: The number of the line to check. - nesting_state: A NestingState instance which maintains information about - the current stack of nested blocks being parsed. - error: The function to call with any errors found. - """ - - # Don't use "elided" lines here, otherwise we can't check commented lines. - # Don't want to use "raw" either, because we don't want to check inside C++11 - # raw strings, - raw = clean_lines.lines_without_raw_strings - line = raw[linenum] - - # Before nixing comments, check if the line is blank for no good - # reason. This includes the first line after a block is opened, and - # blank lines at the end of a function (ie, right before a line like '}' - # - # Skip all the blank line checks if we are immediately inside a - # namespace body. In other words, don't issue blank line warnings - # for this block: - # namespace { - # - # } - # - # A warning about missing end of namespace comments will be issued instead. - # - # Also skip blank line checks for 'extern "C"' blocks, which are formatted - # like namespaces. - if (IsBlankLine(line) and - not nesting_state.InNamespaceBody() and - not nesting_state.InExternC()): - elided = clean_lines.elided - prev_line = elided[linenum - 1] - prevbrace = prev_line.rfind('{') - # TODO(unknown): Don't complain if line before blank line, and line after, - # both start with alnums and are indented the same amount. - # This ignores whitespace at the start of a namespace block - # because those are not usually indented. - if prevbrace != -1 and prev_line[prevbrace:].find('}') == -1: - # OK, we have a blank line at the start of a code block. Before we - # complain, we check if it is an exception to the rule: The previous - # non-empty line has the parameters of a function header that are indented - # 4 spaces (because they did not fit in a 80 column line when placed on - # the same line as the function name). We also check for the case where - # the previous line is indented 6 spaces, which may happen when the - # initializers of a constructor do not fit into a 80 column line. - exception = False - if Match(r' {6}\w', prev_line): # Initializer list? - # We are looking for the opening column of initializer list, which - # should be indented 4 spaces to cause 6 space indentation afterwards. - search_position = linenum-2 - while (search_position >= 0 - and Match(r' {6}\w', elided[search_position])): - search_position -= 1 - exception = (search_position >= 0 - and elided[search_position][:5] == ' :') - else: - # Search for the function arguments or an initializer list. We use a - # simple heuristic here: If the line is indented 4 spaces; and we have a - # closing paren, without the opening paren, followed by an opening brace - # or colon (for initializer lists) we assume that it is the last line of - # a function header. If we have a colon indented 4 spaces, it is an - # initializer list. - exception = (Match(r' {4}\w[^\(]*\)\s*(const\s*)?(\{\s*$|:)', - prev_line) - or Match(r' {4}:', prev_line)) - - if not exception: - error(filename, linenum, 'whitespace/blank_line', 2, - 'Redundant blank line at the start of a code block ' - 'should be deleted.') - # Ignore blank lines at the end of a block in a long if-else - # chain, like this: - # if (condition1) { - # // Something followed by a blank line - # - # } else if (condition2) { - # // Something else - # } - if linenum + 1 < clean_lines.NumLines(): - next_line = raw[linenum + 1] - if (next_line - and Match(r'\s*}', next_line) - and next_line.find('} else ') == -1): - error(filename, linenum, 'whitespace/blank_line', 3, - 'Redundant blank line at the end of a code block ' - 'should be deleted.') - - matched = Match(r'\s*(public|protected|private):', prev_line) - if matched: - error(filename, linenum, 'whitespace/blank_line', 3, - 'Do not leave a blank line after "%s:"' % matched.group(1)) - - # Next, check comments - next_line_start = 0 - if linenum + 1 < clean_lines.NumLines(): - next_line = raw[linenum + 1] - next_line_start = len(next_line) - len(next_line.lstrip()) - CheckComment(line, filename, linenum, next_line_start, error) - - # get rid of comments and strings - line = clean_lines.elided[linenum] - - # You shouldn't have spaces before your brackets, except maybe after - # 'delete []', 'return []() {};', or 'auto [abc, ...] = ...;'. - if Search(r'\w\s+\[', line) and not Search(r'(?:auto&?|delete|return)\s+\[', line): - error(filename, linenum, 'whitespace/braces', 5, - 'Extra space before [') - - # In range-based for, we wanted spaces before and after the colon, but - # not around "::" tokens that might appear. - if (Search(r'for *\(.*[^:]:[^: ]', line) or - Search(r'for *\(.*[^: ]:[^:]', line)): - error(filename, linenum, 'whitespace/forcolon', 2, - 'Missing space around colon in range-based for loop') - - -def CheckOperatorSpacing(filename, clean_lines, linenum, error): - """Checks for horizontal spacing around operators. - - Args: - filename: The name of the current file. - clean_lines: A CleansedLines instance containing the file. - linenum: The number of the line to check. - error: The function to call with any errors found. - """ - line = clean_lines.elided[linenum] - - # Don't try to do spacing checks for operator methods. Do this by - # replacing the troublesome characters with something else, - # preserving column position for all other characters. - # - # The replacement is done repeatedly to avoid false positives from - # operators that call operators. - while True: - match = Match(r'^(.*\boperator\b)(\S+)(\s*\(.*)$', line) - if match: - line = match.group(1) + ('_' * len(match.group(2))) + match.group(3) - else: - break - - # We allow no-spaces around = within an if: "if ( (a=Foo()) == 0 )". - # Otherwise not. Note we only check for non-spaces on *both* sides; - # sometimes people put non-spaces on one side when aligning ='s among - # many lines (not that this is behavior that I approve of...) - if ((Search(r'[\w.]=', line) or - Search(r'=[\w.]', line)) - and not Search(r'\b(if|while|for) ', line) - # Operators taken from [lex.operators] in C++11 standard. - and not Search(r'(>=|<=|==|!=|&=|\^=|\|=|\+=|\*=|\/=|\%=)', line) - and not Search(r'operator=', line)): - error(filename, linenum, 'whitespace/operators', 4, - 'Missing spaces around =') - - # It's ok not to have spaces around binary operators like + - * /, but if - # there's too little whitespace, we get concerned. It's hard to tell, - # though, so we punt on this one for now. TODO. - - # You should always have whitespace around binary operators. - # - # Check <= and >= first to avoid false positives with < and >, then - # check non-include lines for spacing around < and >. - # - # If the operator is followed by a comma, assume it's be used in a - # macro context and don't do any checks. This avoids false - # positives. - # - # Note that && is not included here. This is because there are too - # many false positives due to RValue references. - match = Search(r'[^<>=!\s](==|!=|<=|>=|\|\|)[^<>=!\s,;\)]', line) - if match: - error(filename, linenum, 'whitespace/operators', 3, - 'Missing spaces around %s' % match.group(1)) - elif not Match(r'#.*include', line): - # Look for < that is not surrounded by spaces. This is only - # triggered if both sides are missing spaces, even though - # technically should should flag if at least one side is missing a - # space. This is done to avoid some false positives with shifts. - match = Match(r'^(.*[^\s<])<[^\s=<,]', line) - if match: - (_, _, end_pos) = CloseExpression( - clean_lines, linenum, len(match.group(1))) - if end_pos <= -1: - error(filename, linenum, 'whitespace/operators', 3, - 'Missing spaces around <') - - # Look for > that is not surrounded by spaces. Similar to the - # above, we only trigger if both sides are missing spaces to avoid - # false positives with shifts. - match = Match(r'^(.*[^-\s>])>[^\s=>,]', line) - if match: - (_, _, start_pos) = ReverseCloseExpression( - clean_lines, linenum, len(match.group(1))) - if start_pos <= -1: - error(filename, linenum, 'whitespace/operators', 3, - 'Missing spaces around >') - - # We allow no-spaces around << when used like this: 10<<20, but - # not otherwise (particularly, not when used as streams) - # - # We also allow operators following an opening parenthesis, since - # those tend to be macros that deal with operators. - match = Search(r'(operator|[^\s(<])(?:L|UL|LL|ULL|l|ul|ll|ull)?<<([^\s,=<])', line) - if (match and not (match.group(1).isdigit() and match.group(2).isdigit()) and - not (match.group(1) == 'operator' and match.group(2) == ';')): - error(filename, linenum, 'whitespace/operators', 3, - 'Missing spaces around <<') - - # We allow no-spaces around >> for almost anything. This is because - # C++11 allows ">>" to close nested templates, which accounts for - # most cases when ">>" is not followed by a space. - # - # We still warn on ">>" followed by alpha character, because that is - # likely due to ">>" being used for right shifts, e.g.: - # value >> alpha - # - # When ">>" is used to close templates, the alphanumeric letter that - # follows would be part of an identifier, and there should still be - # a space separating the template type and the identifier. - # type> alpha - match = Search(r'>>[a-zA-Z_]', line) - if match: - error(filename, linenum, 'whitespace/operators', 3, - 'Missing spaces around >>') - - # There shouldn't be space around unary operators - match = Search(r'(!\s|~\s|[\s]--[\s;]|[\s]\+\+[\s;])', line) - if match: - error(filename, linenum, 'whitespace/operators', 4, - 'Extra space for operator %s' % match.group(1)) - - -def CheckParenthesisSpacing(filename, clean_lines, linenum, error): - """Checks for horizontal spacing around parentheses. - - Args: - filename: The name of the current file. - clean_lines: A CleansedLines instance containing the file. - linenum: The number of the line to check. - error: The function to call with any errors found. - """ - line = clean_lines.elided[linenum] - - # No spaces after an if, while, switch, or for - match = Search(r' (if\(|for\(|while\(|switch\()', line) - if match: - error(filename, linenum, 'whitespace/parens', 5, - 'Missing space before ( in %s' % match.group(1)) - - # For if/for/while/switch, the left and right parens should be - # consistent about how many spaces are inside the parens, and - # there should either be zero or one spaces inside the parens. - # We don't want: "if ( foo)" or "if ( foo )". - # Exception: "for ( ; foo; bar)" and "for (foo; bar; )" are allowed. - match = Search(r'\b(if|for|while|switch)\s*' - r'\(([ ]*)(.).*[^ ]+([ ]*)\)\s*{\s*$', - line) - if match: - if len(match.group(2)) != len(match.group(4)): - if not (match.group(3) == ';' and - len(match.group(2)) == 1 + len(match.group(4)) or - not match.group(2) and Search(r'\bfor\s*\(.*; \)', line)): - error(filename, linenum, 'whitespace/parens', 5, - 'Mismatching spaces inside () in %s' % match.group(1)) - if len(match.group(2)) not in [0, 1]: - error(filename, linenum, 'whitespace/parens', 5, - 'Should have zero or one spaces inside ( and ) in %s' % - match.group(1)) - - -def CheckCommaSpacing(filename, clean_lines, linenum, error): - """Checks for horizontal spacing near commas and semicolons. - - Args: - filename: The name of the current file. - clean_lines: A CleansedLines instance containing the file. - linenum: The number of the line to check. - error: The function to call with any errors found. - """ - raw = clean_lines.lines_without_raw_strings - line = clean_lines.elided[linenum] - - # You should always have a space after a comma (either as fn arg or operator) - # - # This does not apply when the non-space character following the - # comma is another comma, since the only time when that happens is - # for empty macro arguments. - # - # We run this check in two passes: first pass on elided lines to - # verify that lines contain missing whitespaces, second pass on raw - # lines to confirm that those missing whitespaces are not due to - # elided comments. - if (Search(r',[^,\s]', ReplaceAll(r'\boperator\s*,\s*\(', 'F(', line)) and - Search(r',[^,\s]', raw[linenum])): - error(filename, linenum, 'whitespace/comma', 3, - 'Missing space after ,') - - # You should always have a space after a semicolon - # except for few corner cases - # TODO(unknown): clarify if 'if (1) { return 1;}' is requires one more - # space after ; - if Search(r';[^\s};\\)/]', line): - error(filename, linenum, 'whitespace/semicolon', 3, - 'Missing space after ;') - - -def _IsType(clean_lines, nesting_state, expr): - """Check if expression looks like a type name, returns true if so. - - Args: - clean_lines: A CleansedLines instance containing the file. - nesting_state: A NestingState instance which maintains information about - the current stack of nested blocks being parsed. - expr: The expression to check. - Returns: - True, if token looks like a type. - """ - # Keep only the last token in the expression - last_word = Match(r'^.*(\b\S+)$', expr) - if last_word: - token = last_word.group(1) - else: - token = expr - - # Match native types and stdint types - if _TYPES.match(token): - return True - - # Try a bit harder to match templated types. Walk up the nesting - # stack until we find something that resembles a typename - # declaration for what we are looking for. - typename_pattern = (r'\b(?:typename|class|struct)\s+' + re.escape(token) + - r'\b') - block_index = len(nesting_state.stack) - 1 - while block_index >= 0: - if isinstance(nesting_state.stack[block_index], _NamespaceInfo): - return False - - # Found where the opening brace is. We want to scan from this - # line up to the beginning of the function, minus a few lines. - # template - # class C - # : public ... { // start scanning here - last_line = nesting_state.stack[block_index].starting_linenum - - next_block_start = 0 - if block_index > 0: - next_block_start = nesting_state.stack[block_index - 1].starting_linenum - first_line = last_line - while first_line >= next_block_start: - if clean_lines.elided[first_line].find('template') >= 0: - break - first_line -= 1 - if first_line < next_block_start: - # Didn't find any "template" keyword before reaching the next block, - # there are probably no template things to check for this block - block_index -= 1 - continue - - # Look for typename in the specified range - for i in xrange(first_line, last_line + 1, 1): - if Search(typename_pattern, clean_lines.elided[i]): - return True - block_index -= 1 - - return False - - -def CheckBracesSpacing(filename, clean_lines, linenum, nesting_state, error): - """Checks for horizontal spacing near commas. - - Args: - filename: The name of the current file. - clean_lines: A CleansedLines instance containing the file. - linenum: The number of the line to check. - nesting_state: A NestingState instance which maintains information about - the current stack of nested blocks being parsed. - error: The function to call with any errors found. - """ - line = clean_lines.elided[linenum] - - # Except after an opening paren, or after another opening brace (in case of - # an initializer list, for instance), you should have spaces before your - # braces when they are delimiting blocks, classes, namespaces etc. - # And since you should never have braces at the beginning of a line, - # this is an easy test. Except that braces used for initialization don't - # follow the same rule; we often don't want spaces before those. - match = Match(r'^(.*[^ ({>]){', line) - - if match: - # Try a bit harder to check for brace initialization. This - # happens in one of the following forms: - # Constructor() : initializer_list_{} { ... } - # Constructor{}.MemberFunction() - # Type variable{}; - # FunctionCall(type{}, ...); - # LastArgument(..., type{}); - # LOG(INFO) << type{} << " ..."; - # map_of_type[{...}] = ...; - # ternary = expr ? new type{} : nullptr; - # OuterTemplate{}> - # - # We check for the character following the closing brace, and - # silence the warning if it's one of those listed above, i.e. - # "{.;,)<>]:". - # - # To account for nested initializer list, we allow any number of - # closing braces up to "{;,)<". We can't simply silence the - # warning on first sight of closing brace, because that would - # cause false negatives for things that are not initializer lists. - # Silence this: But not this: - # Outer{ if (...) { - # Inner{...} if (...){ // Missing space before { - # }; } - # - # There is a false negative with this approach if people inserted - # spurious semicolons, e.g. "if (cond){};", but we will catch the - # spurious semicolon with a separate check. - leading_text = match.group(1) - (endline, endlinenum, endpos) = CloseExpression( - clean_lines, linenum, len(match.group(1))) - trailing_text = '' - if endpos > -1: - trailing_text = endline[endpos:] - for offset in xrange(endlinenum + 1, - min(endlinenum + 3, clean_lines.NumLines() - 1)): - trailing_text += clean_lines.elided[offset] - # We also suppress warnings for `uint64_t{expression}` etc., as the style - # guide recommends brace initialization for integral types to avoid - # overflow/truncation. - if (not Match(r'^[\s}]*[{.;,)<>\]:]', trailing_text) - and not _IsType(clean_lines, nesting_state, leading_text)): - error(filename, linenum, 'whitespace/braces', 5, - 'Missing space before {') - - # Make sure '} else {' has spaces. - if Search(r'}else', line): - error(filename, linenum, 'whitespace/braces', 5, - 'Missing space before else') - - # You shouldn't have a space before a semicolon at the end of the line. - # There's a special case for "for" since the style guide allows space before - # the semicolon there. - if Search(r':\s*;\s*$', line): - error(filename, linenum, 'whitespace/semicolon', 5, - 'Semicolon defining empty statement. Use {} instead.') - elif Search(r'^\s*;\s*$', line): - error(filename, linenum, 'whitespace/semicolon', 5, - 'Line contains only semicolon. If this should be an empty statement, ' - 'use {} instead.') - elif (Search(r'\s+;\s*$', line) and - not Search(r'\bfor\b', line)): - error(filename, linenum, 'whitespace/semicolon', 5, - 'Extra space before last semicolon. If this should be an empty ' - 'statement, use {} instead.') - - -def IsDecltype(clean_lines, linenum, column): - """Check if the token ending on (linenum, column) is decltype(). - - Args: - clean_lines: A CleansedLines instance containing the file. - linenum: the number of the line to check. - column: end column of the token to check. - Returns: - True if this token is decltype() expression, False otherwise. - """ - (text, _, start_col) = ReverseCloseExpression(clean_lines, linenum, column) - if start_col < 0: - return False - if Search(r'\bdecltype\s*$', text[0:start_col]): - return True - return False - -def CheckSectionSpacing(filename, clean_lines, class_info, linenum, error): - """Checks for additional blank line issues related to sections. - - Currently the only thing checked here is blank line before protected/private. - - Args: - filename: The name of the current file. - clean_lines: A CleansedLines instance containing the file. - class_info: A _ClassInfo objects. - linenum: The number of the line to check. - error: The function to call with any errors found. - """ - # Skip checks if the class is small, where small means 25 lines or less. - # 25 lines seems like a good cutoff since that's the usual height of - # terminals, and any class that can't fit in one screen can't really - # be considered "small". - # - # Also skip checks if we are on the first line. This accounts for - # classes that look like - # class Foo { public: ... }; - # - # If we didn't find the end of the class, last_line would be zero, - # and the check will be skipped by the first condition. - if (class_info.last_line - class_info.starting_linenum <= 24 or - linenum <= class_info.starting_linenum): - return - - matched = Match(r'\s*(public|protected|private):', clean_lines.lines[linenum]) - if matched: - # Issue warning if the line before public/protected/private was - # not a blank line, but don't do this if the previous line contains - # "class" or "struct". This can happen two ways: - # - We are at the beginning of the class. - # - We are forward-declaring an inner class that is semantically - # private, but needed to be public for implementation reasons. - # Also ignores cases where the previous line ends with a backslash as can be - # common when defining classes in C macros. - prev_line = clean_lines.lines[linenum - 1] - if (not IsBlankLine(prev_line) and - not Search(r'\b(class|struct)\b', prev_line) and - not Search(r'\\$', prev_line)): - # Try a bit harder to find the beginning of the class. This is to - # account for multi-line base-specifier lists, e.g.: - # class Derived - # : public Base { - end_class_head = class_info.starting_linenum - for i in range(class_info.starting_linenum, linenum): - if Search(r'\{\s*$', clean_lines.lines[i]): - end_class_head = i - break - if end_class_head < linenum - 1: - error(filename, linenum, 'whitespace/blank_line', 3, - '"%s:" should be preceded by a blank line' % matched.group(1)) - - -def GetPreviousNonBlankLine(clean_lines, linenum): - """Return the most recent non-blank line and its line number. - - Args: - clean_lines: A CleansedLines instance containing the file contents. - linenum: The number of the line to check. - - Returns: - A tuple with two elements. The first element is the contents of the last - non-blank line before the current line, or the empty string if this is the - first non-blank line. The second is the line number of that line, or -1 - if this is the first non-blank line. - """ - - prevlinenum = linenum - 1 - while prevlinenum >= 0: - prevline = clean_lines.elided[prevlinenum] - if not IsBlankLine(prevline): # if not a blank line... - return (prevline, prevlinenum) - prevlinenum -= 1 - return ('', -1) - - -def CheckBraces(filename, clean_lines, linenum, error): - """Looks for misplaced braces (e.g. at the end of line). - - Args: - filename: The name of the current file. - clean_lines: A CleansedLines instance containing the file. - linenum: The number of the line to check. - error: The function to call with any errors found. - """ - - line = clean_lines.elided[linenum] # get rid of comments and strings - - if Match(r'\s*{\s*$', line): - # We allow an open brace to start a line in the case where someone is using - # braces in a block to explicitly create a new scope, which is commonly used - # to control the lifetime of stack-allocated variables. Braces are also - # used for brace initializers inside function calls. We don't detect this - # perfectly: we just don't complain if the last non-whitespace character on - # the previous non-blank line is ',', ';', ':', '(', '{', or '}', or if the - # previous line starts a preprocessor block. We also allow a brace on the - # following line if it is part of an array initialization and would not fit - # within the 80 character limit of the preceding line. - prevline = GetPreviousNonBlankLine(clean_lines, linenum)[0] - if (not Search(r'[,;:}{(]\s*$', prevline) and - not Match(r'\s*#', prevline) and - not (GetLineWidth(prevline) > _line_length - 2 and '[]' in prevline)): - error(filename, linenum, 'whitespace/braces', 4, - '{ should almost always be at the end of the previous line') - - # An else clause should be on the same line as the preceding closing brace. - if Match(r'\s*else\b\s*(?:if\b|\{|$)', line): - prevline = GetPreviousNonBlankLine(clean_lines, linenum)[0] - if Match(r'\s*}\s*$', prevline): - error(filename, linenum, 'whitespace/newline', 4, - 'An else should appear on the same line as the preceding }') - - # If braces come on one side of an else, they should be on both. - # However, we have to worry about "else if" that spans multiple lines! - if Search(r'else if\s*\(', line): # could be multi-line if - brace_on_left = bool(Search(r'}\s*else if\s*\(', line)) - # find the ( after the if - pos = line.find('else if') - pos = line.find('(', pos) - if pos > 0: - (endline, _, endpos) = CloseExpression(clean_lines, linenum, pos) - brace_on_right = endline[endpos:].find('{') != -1 - if brace_on_left != brace_on_right: # must be brace after if - error(filename, linenum, 'readability/braces', 5, - 'If an else has a brace on one side, it should have it on both') - elif Search(r'}\s*else[^{]*$', line) or Match(r'[^}]*else\s*{', line): - error(filename, linenum, 'readability/braces', 5, - 'If an else has a brace on one side, it should have it on both') - - # Likewise, an else should never have the else clause on the same line - if Search(r'\belse [^\s{]', line) and not Search(r'\belse if\b', line): - error(filename, linenum, 'whitespace/newline', 4, - 'Else clause should never be on same line as else (use 2 lines)') - - # In the same way, a do/while should never be on one line - if Match(r'\s*do [^\s{]', line): - error(filename, linenum, 'whitespace/newline', 4, - 'do/while clauses should not be on a single line') - - # Check single-line if/else bodies. The style guide says 'curly braces are not - # required for single-line statements'. We additionally allow multi-line, - # single statements, but we reject anything with more than one semicolon in - # it. This means that the first semicolon after the if should be at the end of - # its line, and the line after that should have an indent level equal to or - # lower than the if. We also check for ambiguous if/else nesting without - # braces. - if_else_match = Search(r'\b(if\s*\(|else\b)', line) - if if_else_match and not Match(r'\s*#', line): - if_indent = GetIndentLevel(line) - endline, endlinenum, endpos = line, linenum, if_else_match.end() - if_match = Search(r'\bif\s*\(', line) - if if_match: - # This could be a multiline if condition, so find the end first. - pos = if_match.end() - 1 - (endline, endlinenum, endpos) = CloseExpression(clean_lines, linenum, pos) - # Check for an opening brace, either directly after the if or on the next - # line. If found, this isn't a single-statement conditional. - if (not Match(r'\s*{', endline[endpos:]) - and not (Match(r'\s*$', endline[endpos:]) - and endlinenum < (len(clean_lines.elided) - 1) - and Match(r'\s*{', clean_lines.elided[endlinenum + 1]))): - while (endlinenum < len(clean_lines.elided) - and ';' not in clean_lines.elided[endlinenum][endpos:]): - endlinenum += 1 - endpos = 0 - if endlinenum < len(clean_lines.elided): - endline = clean_lines.elided[endlinenum] - # We allow a mix of whitespace and closing braces (e.g. for one-liner - # methods) and a single \ after the semicolon (for macros) - endpos = endline.find(';') - if not Match(r';[\s}]*(\\?)$', endline[endpos:]): - # Semicolon isn't the last character, there's something trailing. - # Output a warning if the semicolon is not contained inside - # a lambda expression. - if not Match(r'^[^{};]*\[[^\[\]]*\][^{}]*\{[^{}]*\}\s*\)*[;,]\s*$', - endline): - error(filename, linenum, 'readability/braces', 4, - 'If/else bodies with multiple statements require braces') - elif endlinenum < len(clean_lines.elided) - 1: - # Make sure the next line is dedented - next_line = clean_lines.elided[endlinenum + 1] - next_indent = GetIndentLevel(next_line) - # With ambiguous nested if statements, this will error out on the - # if that *doesn't* match the else, regardless of whether it's the - # inner one or outer one. - if (if_match and Match(r'\s*else\b', next_line) - and next_indent != if_indent): - error(filename, linenum, 'readability/braces', 4, - 'Else clause should be indented at the same level as if. ' - 'Ambiguous nested if/else chains require braces.') - elif next_indent > if_indent: - error(filename, linenum, 'readability/braces', 4, - 'If/else bodies with multiple statements require braces') - - -def CheckTrailingSemicolon(filename, clean_lines, linenum, error): - """Looks for redundant trailing semicolon. - - Args: - filename: The name of the current file. - clean_lines: A CleansedLines instance containing the file. - linenum: The number of the line to check. - error: The function to call with any errors found. - """ - - line = clean_lines.elided[linenum] - - # Block bodies should not be followed by a semicolon. Due to C++11 - # brace initialization, there are more places where semicolons are - # required than not, so we use a whitelist approach to check these - # rather than a blacklist. These are the places where "};" should - # be replaced by just "}": - # 1. Some flavor of block following closing parenthesis: - # for (;;) {}; - # while (...) {}; - # switch (...) {}; - # Function(...) {}; - # if (...) {}; - # if (...) else if (...) {}; - # - # 2. else block: - # if (...) else {}; - # - # 3. const member function: - # Function(...) const {}; - # - # 4. Block following some statement: - # x = 42; - # {}; - # - # 5. Block at the beginning of a function: - # Function(...) { - # {}; - # } - # - # Note that naively checking for the preceding "{" will also match - # braces inside multi-dimensional arrays, but this is fine since - # that expression will not contain semicolons. - # - # 6. Block following another block: - # while (true) {} - # {}; - # - # 7. End of namespaces: - # namespace {}; - # - # These semicolons seems far more common than other kinds of - # redundant semicolons, possibly due to people converting classes - # to namespaces. For now we do not warn for this case. - # - # Try matching case 1 first. - match = Match(r'^(.*\)\s*)\{', line) - if match: - # Matched closing parenthesis (case 1). Check the token before the - # matching opening parenthesis, and don't warn if it looks like a - # macro. This avoids these false positives: - # - macro that defines a base class - # - multi-line macro that defines a base class - # - macro that defines the whole class-head - # - # But we still issue warnings for macros that we know are safe to - # warn, specifically: - # - TEST, TEST_F, TEST_P, MATCHER, MATCHER_P - # - TYPED_TEST - # - INTERFACE_DEF - # - EXCLUSIVE_LOCKS_REQUIRED, SHARED_LOCKS_REQUIRED, LOCKS_EXCLUDED: - # - # We implement a whitelist of safe macros instead of a blacklist of - # unsafe macros, even though the latter appears less frequently in - # google code and would have been easier to implement. This is because - # the downside for getting the whitelist wrong means some extra - # semicolons, while the downside for getting the blacklist wrong - # would result in compile errors. - # - # In addition to macros, we also don't want to warn on - # - Compound literals - # - Lambdas - # - alignas specifier with anonymous structs - # - decltype - closing_brace_pos = match.group(1).rfind(')') - opening_parenthesis = ReverseCloseExpression( - clean_lines, linenum, closing_brace_pos) - if opening_parenthesis[2] > -1: - line_prefix = opening_parenthesis[0][0:opening_parenthesis[2]] - macro = Search(r'\b([A-Z_][A-Z0-9_]*)\s*$', line_prefix) - func = Match(r'^(.*\])\s*$', line_prefix) - if ((macro and - macro.group(1) not in ( - 'TEST', 'TEST_F', 'MATCHER', 'MATCHER_P', 'TYPED_TEST', - 'EXCLUSIVE_LOCKS_REQUIRED', 'SHARED_LOCKS_REQUIRED', - 'LOCKS_EXCLUDED', 'INTERFACE_DEF')) or - (func and not Search(r'\boperator\s*\[\s*\]', func.group(1))) or - Search(r'\b(?:struct|union)\s+alignas\s*$', line_prefix) or - Search(r'\bdecltype$', line_prefix) or - Search(r'\s+=\s*$', line_prefix)): - match = None - if (match and - opening_parenthesis[1] > 1 and - Search(r'\]\s*$', clean_lines.elided[opening_parenthesis[1] - 1])): - # Multi-line lambda-expression - match = None - - else: - # Try matching cases 2-3. - match = Match(r'^(.*(?:else|\)\s*const)\s*)\{', line) - if not match: - # Try matching cases 4-6. These are always matched on separate lines. - # - # Note that we can't simply concatenate the previous line to the - # current line and do a single match, otherwise we may output - # duplicate warnings for the blank line case: - # if (cond) { - # // blank line - # } - prevline = GetPreviousNonBlankLine(clean_lines, linenum)[0] - if prevline and Search(r'[;{}]\s*$', prevline): - match = Match(r'^(\s*)\{', line) - - # Check matching closing brace - if match: - (endline, endlinenum, endpos) = CloseExpression( - clean_lines, linenum, len(match.group(1))) - if endpos > -1 and Match(r'^\s*;', endline[endpos:]): - # Current {} pair is eligible for semicolon check, and we have found - # the redundant semicolon, output warning here. - # - # Note: because we are scanning forward for opening braces, and - # outputting warnings for the matching closing brace, if there are - # nested blocks with trailing semicolons, we will get the error - # messages in reversed order. - - # We need to check the line forward for NOLINT - raw_lines = clean_lines.raw_lines - ParseNolintSuppressions(filename, raw_lines[endlinenum-1], endlinenum-1, - error) - ParseNolintSuppressions(filename, raw_lines[endlinenum], endlinenum, - error) - - error(filename, endlinenum, 'readability/braces', 4, - "You don't need a ; after a }") - - -def CheckEmptyBlockBody(filename, clean_lines, linenum, error): - """Look for empty loop/conditional body with only a single semicolon. - - Args: - filename: The name of the current file. - clean_lines: A CleansedLines instance containing the file. - linenum: The number of the line to check. - error: The function to call with any errors found. - """ - - # Search for loop keywords at the beginning of the line. Because only - # whitespaces are allowed before the keywords, this will also ignore most - # do-while-loops, since those lines should start with closing brace. - # - # We also check "if" blocks here, since an empty conditional block - # is likely an error. - line = clean_lines.elided[linenum] - matched = Match(r'\s*(for|while|if)\s*\(', line) - if matched: - # Find the end of the conditional expression. - (end_line, end_linenum, end_pos) = CloseExpression( - clean_lines, linenum, line.find('(')) - - # Output warning if what follows the condition expression is a semicolon. - # No warning for all other cases, including whitespace or newline, since we - # have a separate check for semicolons preceded by whitespace. - if end_pos >= 0 and Match(r';', end_line[end_pos:]): - if matched.group(1) == 'if': - error(filename, end_linenum, 'whitespace/empty_conditional_body', 5, - 'Empty conditional bodies should use {}') - else: - error(filename, end_linenum, 'whitespace/empty_loop_body', 5, - 'Empty loop bodies should use {} or continue') - - # Check for if statements that have completely empty bodies (no comments) - # and no else clauses. - if end_pos >= 0 and matched.group(1) == 'if': - # Find the position of the opening { for the if statement. - # Return without logging an error if it has no brackets. - opening_linenum = end_linenum - opening_line_fragment = end_line[end_pos:] - # Loop until EOF or find anything that's not whitespace or opening {. - while not Search(r'^\s*\{', opening_line_fragment): - if Search(r'^(?!\s*$)', opening_line_fragment): - # Conditional has no brackets. - return - opening_linenum += 1 - if opening_linenum == len(clean_lines.elided): - # Couldn't find conditional's opening { or any code before EOF. - return - opening_line_fragment = clean_lines.elided[opening_linenum] - # Set opening_line (opening_line_fragment may not be entire opening line). - opening_line = clean_lines.elided[opening_linenum] - - # Find the position of the closing }. - opening_pos = opening_line_fragment.find('{') - if opening_linenum == end_linenum: - # We need to make opening_pos relative to the start of the entire line. - opening_pos += end_pos - (closing_line, closing_linenum, closing_pos) = CloseExpression( - clean_lines, opening_linenum, opening_pos) - if closing_pos < 0: - return - - # Now construct the body of the conditional. This consists of the portion - # of the opening line after the {, all lines until the closing line, - # and the portion of the closing line before the }. - if (clean_lines.raw_lines[opening_linenum] != - CleanseComments(clean_lines.raw_lines[opening_linenum])): - # Opening line ends with a comment, so conditional isn't empty. - return - if closing_linenum > opening_linenum: - # Opening line after the {. Ignore comments here since we checked above. - bodylist = list(opening_line[opening_pos+1:]) - # All lines until closing line, excluding closing line, with comments. - bodylist.extend(clean_lines.raw_lines[opening_linenum+1:closing_linenum]) - # Closing line before the }. Won't (and can't) have comments. - bodylist.append(clean_lines.elided[closing_linenum][:closing_pos-1]) - body = '\n'.join(bodylist) - else: - # If statement has brackets and fits on a single line. - body = opening_line[opening_pos+1:closing_pos-1] - - # Check if the body is empty - if not _EMPTY_CONDITIONAL_BODY_PATTERN.search(body): - return - # The body is empty. Now make sure there's not an else clause. - current_linenum = closing_linenum - current_line_fragment = closing_line[closing_pos:] - # Loop until EOF or find anything that's not whitespace or else clause. - while Search(r'^\s*$|^(?=\s*else)', current_line_fragment): - if Search(r'^(?=\s*else)', current_line_fragment): - # Found an else clause, so don't log an error. - return - current_linenum += 1 - if current_linenum == len(clean_lines.elided): - break - current_line_fragment = clean_lines.elided[current_linenum] - - # The body is empty and there's no else clause until EOF or other code. - error(filename, end_linenum, 'whitespace/empty_if_body', 4, - ('If statement had no body and no else clause')) - - -def FindCheckMacro(line): - """Find a replaceable CHECK-like macro. - - Args: - line: line to search on. - Returns: - (macro name, start position), or (None, -1) if no replaceable - macro is found. - """ - for macro in _CHECK_MACROS: - i = line.find(macro) - if i >= 0: - # Find opening parenthesis. Do a regular expression match here - # to make sure that we are matching the expected CHECK macro, as - # opposed to some other macro that happens to contain the CHECK - # substring. - matched = Match(r'^(.*\b' + macro + r'\s*)\(', line) - if not matched: - continue - return (macro, len(matched.group(1))) - return (None, -1) - - -def CheckCheck(filename, clean_lines, linenum, error): - """Checks the use of CHECK and EXPECT macros. - - Args: - filename: The name of the current file. - clean_lines: A CleansedLines instance containing the file. - linenum: The number of the line to check. - error: The function to call with any errors found. - """ - - # Decide the set of replacement macros that should be suggested - lines = clean_lines.elided - (check_macro, start_pos) = FindCheckMacro(lines[linenum]) - if not check_macro: - return - - # Find end of the boolean expression by matching parentheses - (last_line, end_line, end_pos) = CloseExpression( - clean_lines, linenum, start_pos) - if end_pos < 0: - return - - # If the check macro is followed by something other than a - # semicolon, assume users will log their own custom error messages - # and don't suggest any replacements. - if not Match(r'\s*;', last_line[end_pos:]): - return - - if linenum == end_line: - expression = lines[linenum][start_pos + 1:end_pos - 1] - else: - expression = lines[linenum][start_pos + 1:] - for i in xrange(linenum + 1, end_line): - expression += lines[i] - expression += last_line[0:end_pos - 1] - - # Parse expression so that we can take parentheses into account. - # This avoids false positives for inputs like "CHECK((a < 4) == b)", - # which is not replaceable by CHECK_LE. - lhs = '' - rhs = '' - operator = None - while expression: - matched = Match(r'^\s*(<<|<<=|>>|>>=|->\*|->|&&|\|\||' - r'==|!=|>=|>|<=|<|\()(.*)$', expression) - if matched: - token = matched.group(1) - if token == '(': - # Parenthesized operand - expression = matched.group(2) - (end, _) = FindEndOfExpressionInLine(expression, 0, ['(']) - if end < 0: - return # Unmatched parenthesis - lhs += '(' + expression[0:end] - expression = expression[end:] - elif token in ('&&', '||'): - # Logical and/or operators. This means the expression - # contains more than one term, for example: - # CHECK(42 < a && a < b); - # - # These are not replaceable with CHECK_LE, so bail out early. - return - elif token in ('<<', '<<=', '>>', '>>=', '->*', '->'): - # Non-relational operator - lhs += token - expression = matched.group(2) - else: - # Relational operator - operator = token - rhs = matched.group(2) - break - else: - # Unparenthesized operand. Instead of appending to lhs one character - # at a time, we do another regular expression match to consume several - # characters at once if possible. Trivial benchmark shows that this - # is more efficient when the operands are longer than a single - # character, which is generally the case. - matched = Match(r'^([^-=!<>()&|]+)(.*)$', expression) - if not matched: - matched = Match(r'^(\s*\S)(.*)$', expression) - if not matched: - break - lhs += matched.group(1) - expression = matched.group(2) - - # Only apply checks if we got all parts of the boolean expression - if not (lhs and operator and rhs): - return - - # Check that rhs do not contain logical operators. We already know - # that lhs is fine since the loop above parses out && and ||. - if rhs.find('&&') > -1 or rhs.find('||') > -1: - return - - # At least one of the operands must be a constant literal. This is - # to avoid suggesting replacements for unprintable things like - # CHECK(variable != iterator) - # - # The following pattern matches decimal, hex integers, strings, and - # characters (in that order). - lhs = lhs.strip() - rhs = rhs.strip() - match_constant = r'^([-+]?(\d+|0[xX][0-9a-fA-F]+)[lLuU]{0,3}|".*"|\'.*\')$' - if Match(match_constant, lhs) or Match(match_constant, rhs): - # Note: since we know both lhs and rhs, we can provide a more - # descriptive error message like: - # Consider using CHECK_EQ(x, 42) instead of CHECK(x == 42) - # Instead of: - # Consider using CHECK_EQ instead of CHECK(a == b) - # - # We are still keeping the less descriptive message because if lhs - # or rhs gets long, the error message might become unreadable. - error(filename, linenum, 'readability/check', 2, - 'Consider using %s instead of %s(a %s b)' % ( - _CHECK_REPLACEMENT[check_macro][operator], - check_macro, operator)) - - -def CheckAltTokens(filename, clean_lines, linenum, error): - """Check alternative keywords being used in boolean expressions. - - Args: - filename: The name of the current file. - clean_lines: A CleansedLines instance containing the file. - linenum: The number of the line to check. - error: The function to call with any errors found. - """ - line = clean_lines.elided[linenum] - - # Avoid preprocessor lines - if Match(r'^\s*#', line): - return - - # Last ditch effort to avoid multi-line comments. This will not help - # if the comment started before the current line or ended after the - # current line, but it catches most of the false positives. At least, - # it provides a way to workaround this warning for people who use - # multi-line comments in preprocessor macros. - # - # TODO(unknown): remove this once cpplint has better support for - # multi-line comments. - if line.find('/*') >= 0 or line.find('*/') >= 0: - return - - for match in _ALT_TOKEN_REPLACEMENT_PATTERN.finditer(line): - error(filename, linenum, 'readability/alt_tokens', 2, - 'Use operator %s instead of %s' % ( - _ALT_TOKEN_REPLACEMENT[match.group(1)], match.group(1))) - - -def GetLineWidth(line): - """Determines the width of the line in column positions. - - Args: - line: A string, which may be a Unicode string. - - Returns: - The width of the line in column positions, accounting for Unicode - combining characters and wide characters. - """ - if isinstance(line, unicode): - width = 0 - for uc in unicodedata.normalize('NFC', line): - if unicodedata.east_asian_width(uc) in ('W', 'F'): - width += 2 - elif not unicodedata.combining(uc): - # Issue 337 - # https://mail.python.org/pipermail/python-list/2012-August/628809.html - if (sys.version_info.major, sys.version_info.minor) <= (3, 2): - # https://github.com/python/cpython/blob/2.7/Include/unicodeobject.h#L81 - is_wide_build = sysconfig.get_config_var("Py_UNICODE_SIZE") >= 4 - # https://github.com/python/cpython/blob/2.7/Objects/unicodeobject.c#L564 - is_low_surrogate = 0xDC00 <= ord(uc) <= 0xDFFF - if not is_wide_build and is_low_surrogate: - width -= 1 - - width += 1 - return width - else: - return len(line) - - -def CheckStyle(filename, clean_lines, linenum, file_extension, nesting_state, - error): - """Checks rules from the 'C++ style rules' section of cppguide.html. - - Most of these rules are hard to test (naming, comment style), but we - do what we can. In particular we check for 2-space indents, line lengths, - tab usage, spaces inside code, etc. - - Args: - filename: The name of the current file. - clean_lines: A CleansedLines instance containing the file. - linenum: The number of the line to check. - file_extension: The extension (without the dot) of the filename. - nesting_state: A NestingState instance which maintains information about - the current stack of nested blocks being parsed. - error: The function to call with any errors found. - """ - - # Don't use "elided" lines here, otherwise we can't check commented lines. - # Don't want to use "raw" either, because we don't want to check inside C++11 - # raw strings, - raw_lines = clean_lines.lines_without_raw_strings - line = raw_lines[linenum] - prev = raw_lines[linenum - 1] if linenum > 0 else '' - - if line.find('\t') != -1: - error(filename, linenum, 'whitespace/tab', 1, - 'Tab found; better to use spaces') - - # One or three blank spaces at the beginning of the line is weird; it's - # hard to reconcile that with 2-space indents. - # NOTE: here are the conditions rob pike used for his tests. Mine aren't - # as sophisticated, but it may be worth becoming so: RLENGTH==initial_spaces - # if(RLENGTH > 20) complain = 0; - # if(match($0, " +(error|private|public|protected):")) complain = 0; - # if(match(prev, "&& *$")) complain = 0; - # if(match(prev, "\\|\\| *$")) complain = 0; - # if(match(prev, "[\",=><] *$")) complain = 0; - # if(match($0, " <<")) complain = 0; - # if(match(prev, " +for \\(")) complain = 0; - # if(prevodd && match(prevprev, " +for \\(")) complain = 0; - scope_or_label_pattern = r'\s*\w+\s*:\s*\\?$' - classinfo = nesting_state.InnermostClass() - initial_spaces = 0 - cleansed_line = clean_lines.elided[linenum] - while initial_spaces < len(line) and line[initial_spaces] == ' ': - initial_spaces += 1 - # There are certain situations we allow one space, notably for - # section labels, and also lines containing multi-line raw strings. - # We also don't check for lines that look like continuation lines - # (of lines ending in double quotes, commas, equals, or angle brackets) - # because the rules for how to indent those are non-trivial. - if (not Search(r'[",=><] *$', prev) and - (initial_spaces == 1 or initial_spaces == 3) and - not Match(scope_or_label_pattern, cleansed_line) and - not (clean_lines.raw_lines[linenum] != line and - Match(r'^\s*""', line))): - error(filename, linenum, 'whitespace/indent', 3, - 'Weird number of spaces at line-start. ' - 'Are you using a 2-space indent?') - - if line and line[-1].isspace(): - error(filename, linenum, 'whitespace/end_of_line', 4, - 'Line ends in whitespace. Consider deleting these extra spaces.') - - # Check if the line is a header guard. - is_header_guard = False - if IsHeaderExtension(file_extension): - cppvar = GetHeaderGuardCPPVariable(filename) - if (line.startswith('#ifndef %s' % cppvar) or - line.startswith('#define %s' % cppvar) or - line.startswith('#endif // %s' % cppvar)): - is_header_guard = True - # #include lines and header guards can be long, since there's no clean way to - # split them. - # - # URLs can be long too. It's possible to split these, but it makes them - # harder to cut&paste. - # - # The "$Id:...$" comment may also get very long without it being the - # developers fault. - # - # Doxygen documentation copying can get pretty long when using an overloaded - # function declaration - if (not line.startswith('#include') and not is_header_guard and - not Match(r'^\s*//.*http(s?)://\S*$', line) and - not Match(r'^\s*//\s*[^\s]*$', line) and - not Match(r'^// \$Id:.*#[0-9]+ \$$', line) and - not Match(r'^\s*/// [@\\](copydoc|copydetails|copybrief) .*$', line)): - line_width = GetLineWidth(line) - if line_width > _line_length: - error(filename, linenum, 'whitespace/line_length', 2, - 'Lines should be <= %i characters long' % _line_length) - - if (cleansed_line.count(';') > 1 and - # allow simple single line lambdas - not Match(r'^[^{};]*\[[^\[\]]*\][^{}]*\{[^{}\n\r]*\}', - line) and - # for loops are allowed two ;'s (and may run over two lines). - cleansed_line.find('for') == -1 and - (GetPreviousNonBlankLine(clean_lines, linenum)[0].find('for') == -1 or - GetPreviousNonBlankLine(clean_lines, linenum)[0].find(';') != -1) and - # It's ok to have many commands in a switch case that fits in 1 line - not ((cleansed_line.find('case ') != -1 or - cleansed_line.find('default:') != -1) and - cleansed_line.find('break;') != -1)): - error(filename, linenum, 'whitespace/newline', 0, - 'More than one command on the same line') - - # Some more style checks - CheckBraces(filename, clean_lines, linenum, error) - CheckTrailingSemicolon(filename, clean_lines, linenum, error) - CheckEmptyBlockBody(filename, clean_lines, linenum, error) - CheckSpacing(filename, clean_lines, linenum, nesting_state, error) - CheckOperatorSpacing(filename, clean_lines, linenum, error) - CheckParenthesisSpacing(filename, clean_lines, linenum, error) - CheckCommaSpacing(filename, clean_lines, linenum, error) - CheckBracesSpacing(filename, clean_lines, linenum, nesting_state, error) - CheckSpacingForFunctionCall(filename, clean_lines, linenum, error) - CheckCheck(filename, clean_lines, linenum, error) - CheckAltTokens(filename, clean_lines, linenum, error) - classinfo = nesting_state.InnermostClass() - if classinfo: - CheckSectionSpacing(filename, clean_lines, classinfo, linenum, error) - - -_RE_PATTERN_INCLUDE = re.compile(r'^\s*#\s*include\s*([<"])([^>"]*)[>"].*$') -# Matches the first component of a filename delimited by -s and _s. That is: -# _RE_FIRST_COMPONENT.match('foo').group(0) == 'foo' -# _RE_FIRST_COMPONENT.match('foo.cc').group(0) == 'foo' -# _RE_FIRST_COMPONENT.match('foo-bar_baz.cc').group(0) == 'foo' -# _RE_FIRST_COMPONENT.match('foo_bar-baz.cc').group(0) == 'foo' -_RE_FIRST_COMPONENT = re.compile(r'^[^-_.]+') - - -def _DropCommonSuffixes(filename): - """Drops common suffixes like _test.cc or -inl.h from filename. - - For example: - >>> _DropCommonSuffixes('foo/foo-inl.h') - 'foo/foo' - >>> _DropCommonSuffixes('foo/bar/foo.cc') - 'foo/bar/foo' - >>> _DropCommonSuffixes('foo/foo_internal.h') - 'foo/foo' - >>> _DropCommonSuffixes('foo/foo_unusualinternal.h') - 'foo/foo_unusualinternal' - - Args: - filename: The input filename. - - Returns: - The filename with the common suffix removed. - """ - for suffix in itertools.chain( - ('%s.%s' % (test_suffix.lstrip('_'), ext) - for test_suffix, ext in itertools.product(_test_suffixes, GetNonHeaderExtensions())), - ('%s.%s' % (suffix, ext) - for suffix, ext in itertools.product(['inl', 'imp', 'internal'], GetHeaderExtensions()))): - if (filename.endswith(suffix) and len(filename) > len(suffix) and - filename[-len(suffix) - 1] in ('-', '_')): - return filename[:-len(suffix) - 1] - return os.path.splitext(filename)[0] - - -def _ClassifyInclude(fileinfo, include, is_system): - """Figures out what kind of header 'include' is. - - Args: - fileinfo: The current file cpplint is running over. A FileInfo instance. - include: The path to a #included file. - is_system: True if the #include used <> rather than "". - - Returns: - One of the _XXX_HEADER constants. - - For example: - >>> _ClassifyInclude(FileInfo('foo/foo.cc'), 'stdio.h', True) - _C_SYS_HEADER - >>> _ClassifyInclude(FileInfo('foo/foo.cc'), 'string', True) - _CPP_SYS_HEADER - >>> _ClassifyInclude(FileInfo('foo/foo.cc'), 'foo/foo.h', False) - _LIKELY_MY_HEADER - >>> _ClassifyInclude(FileInfo('foo/foo_unknown_extension.cc'), - ... 'bar/foo_other_ext.h', False) - _POSSIBLE_MY_HEADER - >>> _ClassifyInclude(FileInfo('foo/foo.cc'), 'foo/bar.h', False) - _OTHER_HEADER - """ - # This is a list of all standard c++ header files, except - # those already checked for above. - is_cpp_h = include in _CPP_HEADERS - - # Headers with C++ extensions shouldn't be considered C system headers - if is_system and os.path.splitext(include)[1] in ['.hpp', '.hxx', '.h++']: - is_system = False - - if is_system: - if is_cpp_h: - return _CPP_SYS_HEADER - else: - return _C_SYS_HEADER - - # If the target file and the include we're checking share a - # basename when we drop common extensions, and the include - # lives in . , then it's likely to be owned by the target file. - target_dir, target_base = ( - os.path.split(_DropCommonSuffixes(fileinfo.RepositoryName()))) - include_dir, include_base = os.path.split(_DropCommonSuffixes(include)) - target_dir_pub = os.path.normpath(target_dir + '/../public') - target_dir_pub = target_dir_pub.replace('\\', '/') - if target_base == include_base and ( - include_dir == target_dir or - include_dir == target_dir_pub): - return _LIKELY_MY_HEADER - - # If the target and include share some initial basename - # component, it's possible the target is implementing the - # include, so it's allowed to be first, but we'll never - # complain if it's not there. - target_first_component = _RE_FIRST_COMPONENT.match(target_base) - include_first_component = _RE_FIRST_COMPONENT.match(include_base) - if (target_first_component and include_first_component and - target_first_component.group(0) == - include_first_component.group(0)): - return _POSSIBLE_MY_HEADER - - return _OTHER_HEADER - - - -def CheckIncludeLine(filename, clean_lines, linenum, include_state, error): - """Check rules that are applicable to #include lines. - - Strings on #include lines are NOT removed from elided line, to make - certain tasks easier. However, to prevent false positives, checks - applicable to #include lines in CheckLanguage must be put here. - - Args: - filename: The name of the current file. - clean_lines: A CleansedLines instance containing the file. - linenum: The number of the line to check. - include_state: An _IncludeState instance in which the headers are inserted. - error: The function to call with any errors found. - """ - fileinfo = FileInfo(filename) - line = clean_lines.lines[linenum] - - # "include" should use the new style "foo/bar.h" instead of just "bar.h" - # Only do this check if the included header follows google naming - # conventions. If not, assume that it's a 3rd party API that - # requires special include conventions. - # - # We also make an exception for Lua headers, which follow google - # naming convention but not the include convention. - match = Match(r'#include\s*"([^/]+\.h)"', line) - if match and not _THIRD_PARTY_HEADERS_PATTERN.match(match.group(1)): - error(filename, linenum, 'build/include_subdir', 4, - 'Include the directory when naming .h files') - - # we shouldn't include a file more than once. actually, there are a - # handful of instances where doing so is okay, but in general it's - # not. - match = _RE_PATTERN_INCLUDE.search(line) - if match: - include = match.group(2) - is_system = (match.group(1) == '<') - duplicate_line = include_state.FindHeader(include) - if duplicate_line >= 0: - error(filename, linenum, 'build/include', 4, - '"%s" already included at %s:%s' % - (include, filename, duplicate_line)) - return - - for extension in GetNonHeaderExtensions(): - if (include.endswith('.' + extension) and - os.path.dirname(fileinfo.RepositoryName()) != os.path.dirname(include)): - error(filename, linenum, 'build/include', 4, - 'Do not include .' + extension + ' files from other packages') - return - - # We DO want to include a 3rd party looking header if it matches the - # filename. Otherwise we get an erroneous error "...should include its - # header" error later. - third_src_header = False - for ext in GetHeaderExtensions(): - basefilename = filename[0:len(filename) - len(fileinfo.Extension())] - headerfile = basefilename + '.' + ext - headername = FileInfo(headerfile).RepositoryName() - if headername in include or include in headername: - third_src_header = True - break - - if third_src_header or not _THIRD_PARTY_HEADERS_PATTERN.match(include): - include_state.include_list[-1].append((include, linenum)) - - # We want to ensure that headers appear in the right order: - # 1) for foo.cc, foo.h (preferred location) - # 2) c system files - # 3) cpp system files - # 4) for foo.cc, foo.h (deprecated location) - # 5) other google headers - # - # We classify each include statement as one of those 5 types - # using a number of techniques. The include_state object keeps - # track of the highest type seen, and complains if we see a - # lower type after that. - error_message = include_state.CheckNextIncludeOrder( - _ClassifyInclude(fileinfo, include, is_system)) - if error_message: - error(filename, linenum, 'build/include_order', 4, - '%s. Should be: %s.h, c system, c++ system, other.' % - (error_message, fileinfo.BaseName())) - canonical_include = include_state.CanonicalizeAlphabeticalOrder(include) - if not include_state.IsInAlphabeticalOrder( - clean_lines, linenum, canonical_include): - error(filename, linenum, 'build/include_alpha', 4, - 'Include "%s" not in alphabetical order' % include) - include_state.SetLastHeader(canonical_include) - - - -def _GetTextInside(text, start_pattern): - r"""Retrieves all the text between matching open and close parentheses. - - Given a string of lines and a regular expression string, retrieve all the text - following the expression and between opening punctuation symbols like - (, [, or {, and the matching close-punctuation symbol. This properly nested - occurrences of the punctuations, so for the text like - printf(a(), b(c())); - a call to _GetTextInside(text, r'printf\(') will return 'a(), b(c())'. - start_pattern must match string having an open punctuation symbol at the end. - - Args: - text: The lines to extract text. Its comments and strings must be elided. - It can be single line and can span multiple lines. - start_pattern: The regexp string indicating where to start extracting - the text. - Returns: - The extracted text. - None if either the opening string or ending punctuation could not be found. - """ - # TODO(unknown): Audit cpplint.py to see what places could be profitably - # rewritten to use _GetTextInside (and use inferior regexp matching today). - - # Give opening punctuations to get the matching close-punctuations. - matching_punctuation = {'(': ')', '{': '}', '[': ']'} - closing_punctuation = set(itervalues(matching_punctuation)) - - # Find the position to start extracting text. - match = re.search(start_pattern, text, re.M) - if not match: # start_pattern not found in text. - return None - start_position = match.end(0) - - assert start_position > 0, ( - 'start_pattern must ends with an opening punctuation.') - assert text[start_position - 1] in matching_punctuation, ( - 'start_pattern must ends with an opening punctuation.') - # Stack of closing punctuations we expect to have in text after position. - punctuation_stack = [matching_punctuation[text[start_position - 1]]] - position = start_position - while punctuation_stack and position < len(text): - if text[position] == punctuation_stack[-1]: - punctuation_stack.pop() - elif text[position] in closing_punctuation: - # A closing punctuation without matching opening punctuations. - return None - elif text[position] in matching_punctuation: - punctuation_stack.append(matching_punctuation[text[position]]) - position += 1 - if punctuation_stack: - # Opening punctuations left without matching close-punctuations. - return None - # punctuations match. - return text[start_position:position - 1] - - -# Patterns for matching call-by-reference parameters. -# -# Supports nested templates up to 2 levels deep using this messy pattern: -# < (?: < (?: < [^<>]* -# > -# | [^<>] )* -# > -# | [^<>] )* -# > -_RE_PATTERN_IDENT = r'[_a-zA-Z]\w*' # =~ [[:alpha:]][[:alnum:]]* -_RE_PATTERN_TYPE = ( - r'(?:const\s+)?(?:typename\s+|class\s+|struct\s+|union\s+|enum\s+)?' - r'(?:\w|' - r'\s*<(?:<(?:<[^<>]*>|[^<>])*>|[^<>])*>|' - r'::)+') -# A call-by-reference parameter ends with '& identifier'. -_RE_PATTERN_REF_PARAM = re.compile( - r'(' + _RE_PATTERN_TYPE + r'(?:\s*(?:\bconst\b|[*]))*\s*' - r'&\s*' + _RE_PATTERN_IDENT + r')\s*(?:=[^,()]+)?[,)]') -# A call-by-const-reference parameter either ends with 'const& identifier' -# or looks like 'const type& identifier' when 'type' is atomic. -_RE_PATTERN_CONST_REF_PARAM = ( - r'(?:.*\s*\bconst\s*&\s*' + _RE_PATTERN_IDENT + - r'|const\s+' + _RE_PATTERN_TYPE + r'\s*&\s*' + _RE_PATTERN_IDENT + r')') -# Stream types. -_RE_PATTERN_REF_STREAM_PARAM = ( - r'(?:.*stream\s*&\s*' + _RE_PATTERN_IDENT + r')') - - -def CheckLanguage(filename, clean_lines, linenum, file_extension, - include_state, nesting_state, error): - """Checks rules from the 'C++ language rules' section of cppguide.html. - - Some of these rules are hard to test (function overloading, using - uint32 inappropriately), but we do the best we can. - - Args: - filename: The name of the current file. - clean_lines: A CleansedLines instance containing the file. - linenum: The number of the line to check. - file_extension: The extension (without the dot) of the filename. - include_state: An _IncludeState instance in which the headers are inserted. - nesting_state: A NestingState instance which maintains information about - the current stack of nested blocks being parsed. - error: The function to call with any errors found. - """ - # If the line is empty or consists of entirely a comment, no need to - # check it. - line = clean_lines.elided[linenum] - if not line: - return - - match = _RE_PATTERN_INCLUDE.search(line) - if match: - CheckIncludeLine(filename, clean_lines, linenum, include_state, error) - return - - # Reset include state across preprocessor directives. This is meant - # to silence warnings for conditional includes. - match = Match(r'^\s*#\s*(if|ifdef|ifndef|elif|else|endif)\b', line) - if match: - include_state.ResetSection(match.group(1)) - - - # Perform other checks now that we are sure that this is not an include line - CheckCasts(filename, clean_lines, linenum, error) - CheckGlobalStatic(filename, clean_lines, linenum, error) - CheckPrintf(filename, clean_lines, linenum, error) - - if IsHeaderExtension(file_extension): - # TODO(unknown): check that 1-arg constructors are explicit. - # How to tell it's a constructor? - # (handled in CheckForNonStandardConstructs for now) - # TODO(unknown): check that classes declare or disable copy/assign - # (level 1 error) - pass - - # Check if people are using the verboten C basic types. The only exception - # we regularly allow is "unsigned short port" for port. - if Search(r'\bshort port\b', line): - if not Search(r'\bunsigned short port\b', line): - error(filename, linenum, 'runtime/int', 4, - 'Use "unsigned short" for ports, not "short"') - else: - match = Search(r'\b(short|long(?! +double)|long long)\b', line) - if match: - error(filename, linenum, 'runtime/int', 4, - 'Use int16/int64/etc, rather than the C type %s' % match.group(1)) - - # Check if some verboten operator overloading is going on - # TODO(unknown): catch out-of-line unary operator&: - # class X {}; - # int operator&(const X& x) { return 42; } // unary operator& - # The trick is it's hard to tell apart from binary operator&: - # class Y { int operator&(const Y& x) { return 23; } }; // binary operator& - if Search(r'\boperator\s*&\s*\(\s*\)', line): - error(filename, linenum, 'runtime/operator', 4, - 'Unary operator& is dangerous. Do not use it.') - - # Check for suspicious usage of "if" like - # } if (a == b) { - if Search(r'\}\s*if\s*\(', line): - error(filename, linenum, 'readability/braces', 4, - 'Did you mean "else if"? If not, start a new line for "if".') - - # Check for potential format string bugs like printf(foo). - # We constrain the pattern not to pick things like DocidForPrintf(foo). - # Not perfect but it can catch printf(foo.c_str()) and printf(foo->c_str()) - # TODO(unknown): Catch the following case. Need to change the calling - # convention of the whole function to process multiple line to handle it. - # printf( - # boy_this_is_a_really_long_variable_that_cannot_fit_on_the_prev_line); - printf_args = _GetTextInside(line, r'(?i)\b(string)?printf\s*\(') - if printf_args: - match = Match(r'([\w.\->()]+)$', printf_args) - if match and match.group(1) != '__VA_ARGS__': - function_name = re.search(r'\b((?:string)?printf)\s*\(', - line, re.I).group(1) - error(filename, linenum, 'runtime/printf', 4, - 'Potential format string bug. Do %s("%%s", %s) instead.' - % (function_name, match.group(1))) - - # Check for potential memset bugs like memset(buf, sizeof(buf), 0). - match = Search(r'memset\s*\(([^,]*),\s*([^,]*),\s*0\s*\)', line) - if match and not Match(r"^''|-?[0-9]+|0x[0-9A-Fa-f]$", match.group(2)): - error(filename, linenum, 'runtime/memset', 4, - 'Did you mean "memset(%s, 0, %s)"?' - % (match.group(1), match.group(2))) - - if Search(r'\busing namespace\b', line): - if Search(r'\bliterals\b', line): - error(filename, linenum, 'build/namespaces_literals', 5, - 'Do not use namespace using-directives. ' - 'Use using-declarations instead.') - else: - error(filename, linenum, 'build/namespaces', 5, - 'Do not use namespace using-directives. ' - 'Use using-declarations instead.') - - # Detect variable-length arrays. - match = Match(r'\s*(.+::)?(\w+) [a-z]\w*\[(.+)];', line) - if (match and match.group(2) != 'return' and match.group(2) != 'delete' and - match.group(3).find(']') == -1): - # Split the size using space and arithmetic operators as delimiters. - # If any of the resulting tokens are not compile time constants then - # report the error. - tokens = re.split(r'\s|\+|\-|\*|\/|<<|>>]', match.group(3)) - is_const = True - skip_next = False - for tok in tokens: - if skip_next: - skip_next = False - continue - - if Search(r'sizeof\(.+\)', tok): continue - if Search(r'arraysize\(\w+\)', tok): continue - - tok = tok.lstrip('(') - tok = tok.rstrip(')') - if not tok: continue - if Match(r'\d+', tok): continue - if Match(r'0[xX][0-9a-fA-F]+', tok): continue - if Match(r'k[A-Z0-9]\w*', tok): continue - if Match(r'(.+::)?k[A-Z0-9]\w*', tok): continue - if Match(r'(.+::)?[A-Z][A-Z0-9_]*', tok): continue - # A catch all for tricky sizeof cases, including 'sizeof expression', - # 'sizeof(*type)', 'sizeof(const type)', 'sizeof(struct StructName)' - # requires skipping the next token because we split on ' ' and '*'. - if tok.startswith('sizeof'): - skip_next = True - continue - is_const = False - break - if not is_const: - error(filename, linenum, 'runtime/arrays', 1, - 'Do not use variable-length arrays. Use an appropriately named ' - "('k' followed by CamelCase) compile-time constant for the size.") - - # Check for use of unnamed namespaces in header files. Registration - # macros are typically OK, so we allow use of "namespace {" on lines - # that end with backslashes. - if (IsHeaderExtension(file_extension) - and Search(r'\bnamespace\s*{', line) - and line[-1] != '\\'): - error(filename, linenum, 'build/namespaces', 4, - 'Do not use unnamed namespaces in header files. See ' - 'https://google-styleguide.googlecode.com/svn/trunk/cppguide.xml#Namespaces' - ' for more information.') - - -def CheckGlobalStatic(filename, clean_lines, linenum, error): - """Check for unsafe global or static objects. - - Args: - filename: The name of the current file. - clean_lines: A CleansedLines instance containing the file. - linenum: The number of the line to check. - error: The function to call with any errors found. - """ - line = clean_lines.elided[linenum] - - # Match two lines at a time to support multiline declarations - if linenum + 1 < clean_lines.NumLines() and not Search(r'[;({]', line): - line += clean_lines.elided[linenum + 1].strip() - - # Check for people declaring static/global STL strings at the top level. - # This is dangerous because the C++ language does not guarantee that - # globals with constructors are initialized before the first access, and - # also because globals can be destroyed when some threads are still running. - # TODO(unknown): Generalize this to also find static unique_ptr instances. - # TODO(unknown): File bugs for clang-tidy to find these. - match = Match( - r'((?:|static +)(?:|const +))(?::*std::)?string( +const)? +' - r'([a-zA-Z0-9_:]+)\b(.*)', - line) - - # Remove false positives: - # - String pointers (as opposed to values). - # string *pointer - # const string *pointer - # string const *pointer - # string *const pointer - # - # - Functions and template specializations. - # string Function(... - # string Class::Method(... - # - # - Operators. These are matched separately because operator names - # cross non-word boundaries, and trying to match both operators - # and functions at the same time would decrease accuracy of - # matching identifiers. - # string Class::operator*() - if (match and - not Search(r'\bstring\b(\s+const)?\s*[\*\&]\s*(const\s+)?\w', line) and - not Search(r'\boperator\W', line) and - not Match(r'\s*(<.*>)?(::[a-zA-Z0-9_]+)*\s*\(([^"]|$)', match.group(4))): - if Search(r'\bconst\b', line): - error(filename, linenum, 'runtime/string', 4, - 'For a static/global string constant, use a C style string ' - 'instead: "%schar%s %s[]".' % - (match.group(1), match.group(2) or '', match.group(3))) - else: - error(filename, linenum, 'runtime/string', 4, - 'Static/global string variables are not permitted.') - - if (Search(r'\b([A-Za-z0-9_]*_)\(\1\)', line) or - Search(r'\b([A-Za-z0-9_]*_)\(CHECK_NOTNULL\(\1\)\)', line)): - error(filename, linenum, 'runtime/init', 4, - 'You seem to be initializing a member variable with itself.') - - -def CheckPrintf(filename, clean_lines, linenum, error): - """Check for printf related issues. - - Args: - filename: The name of the current file. - clean_lines: A CleansedLines instance containing the file. - linenum: The number of the line to check. - error: The function to call with any errors found. - """ - line = clean_lines.elided[linenum] - - # When snprintf is used, the second argument shouldn't be a literal. - match = Search(r'snprintf\s*\(([^,]*),\s*([0-9]*)\s*,', line) - if match and match.group(2) != '0': - # If 2nd arg is zero, snprintf is used to calculate size. - error(filename, linenum, 'runtime/printf', 3, - 'If you can, use sizeof(%s) instead of %s as the 2nd arg ' - 'to snprintf.' % (match.group(1), match.group(2))) - - # Check if some verboten C functions are being used. - if Search(r'\bsprintf\s*\(', line): - error(filename, linenum, 'runtime/printf', 5, - 'Never use sprintf. Use snprintf instead.') - match = Search(r'\b(strcpy|strcat)\s*\(', line) - if match: - error(filename, linenum, 'runtime/printf', 4, - 'Almost always, snprintf is better than %s' % match.group(1)) - - -def IsDerivedFunction(clean_lines, linenum): - """Check if current line contains an inherited function. - - Args: - clean_lines: A CleansedLines instance containing the file. - linenum: The number of the line to check. - Returns: - True if current line contains a function with "override" - virt-specifier. - """ - # Scan back a few lines for start of current function - for i in xrange(linenum, max(-1, linenum - 10), -1): - match = Match(r'^([^()]*\w+)\(', clean_lines.elided[i]) - if match: - # Look for "override" after the matching closing parenthesis - line, _, closing_paren = CloseExpression( - clean_lines, i, len(match.group(1))) - return (closing_paren >= 0 and - Search(r'\boverride\b', line[closing_paren:])) - return False - - -def IsOutOfLineMethodDefinition(clean_lines, linenum): - """Check if current line contains an out-of-line method definition. - - Args: - clean_lines: A CleansedLines instance containing the file. - linenum: The number of the line to check. - Returns: - True if current line contains an out-of-line method definition. - """ - # Scan back a few lines for start of current function - for i in xrange(linenum, max(-1, linenum - 10), -1): - if Match(r'^([^()]*\w+)\(', clean_lines.elided[i]): - return Match(r'^[^()]*\w+::\w+\(', clean_lines.elided[i]) is not None - return False - - -def IsInitializerList(clean_lines, linenum): - """Check if current line is inside constructor initializer list. - - Args: - clean_lines: A CleansedLines instance containing the file. - linenum: The number of the line to check. - Returns: - True if current line appears to be inside constructor initializer - list, False otherwise. - """ - for i in xrange(linenum, 1, -1): - line = clean_lines.elided[i] - if i == linenum: - remove_function_body = Match(r'^(.*)\{\s*$', line) - if remove_function_body: - line = remove_function_body.group(1) - - if Search(r'\s:\s*\w+[({]', line): - # A lone colon tend to indicate the start of a constructor - # initializer list. It could also be a ternary operator, which - # also tend to appear in constructor initializer lists as - # opposed to parameter lists. - return True - if Search(r'\}\s*,\s*$', line): - # A closing brace followed by a comma is probably the end of a - # brace-initialized member in constructor initializer list. - return True - if Search(r'[{};]\s*$', line): - # Found one of the following: - # - A closing brace or semicolon, probably the end of the previous - # function. - # - An opening brace, probably the start of current class or namespace. - # - # Current line is probably not inside an initializer list since - # we saw one of those things without seeing the starting colon. - return False - - # Got to the beginning of the file without seeing the start of - # constructor initializer list. - return False - - -def CheckForNonConstReference(filename, clean_lines, linenum, - nesting_state, error): - """Check for non-const references. - - Separate from CheckLanguage since it scans backwards from current - line, instead of scanning forward. - - Args: - filename: The name of the current file. - clean_lines: A CleansedLines instance containing the file. - linenum: The number of the line to check. - nesting_state: A NestingState instance which maintains information about - the current stack of nested blocks being parsed. - error: The function to call with any errors found. - """ - # Do nothing if there is no '&' on current line. - line = clean_lines.elided[linenum] - if '&' not in line: - return - - # If a function is inherited, current function doesn't have much of - # a choice, so any non-const references should not be blamed on - # derived function. - if IsDerivedFunction(clean_lines, linenum): - return - - # Don't warn on out-of-line method definitions, as we would warn on the - # in-line declaration, if it isn't marked with 'override'. - if IsOutOfLineMethodDefinition(clean_lines, linenum): - return - - # Long type names may be broken across multiple lines, usually in one - # of these forms: - # LongType - # ::LongTypeContinued &identifier - # LongType:: - # LongTypeContinued &identifier - # LongType< - # ...>::LongTypeContinued &identifier - # - # If we detected a type split across two lines, join the previous - # line to current line so that we can match const references - # accordingly. - # - # Note that this only scans back one line, since scanning back - # arbitrary number of lines would be expensive. If you have a type - # that spans more than 2 lines, please use a typedef. - if linenum > 1: - previous = None - if Match(r'\s*::(?:[\w<>]|::)+\s*&\s*\S', line): - # previous_line\n + ::current_line - previous = Search(r'\b((?:const\s*)?(?:[\w<>]|::)+[\w<>])\s*$', - clean_lines.elided[linenum - 1]) - elif Match(r'\s*[a-zA-Z_]([\w<>]|::)+\s*&\s*\S', line): - # previous_line::\n + current_line - previous = Search(r'\b((?:const\s*)?(?:[\w<>]|::)+::)\s*$', - clean_lines.elided[linenum - 1]) - if previous: - line = previous.group(1) + line.lstrip() - else: - # Check for templated parameter that is split across multiple lines - endpos = line.rfind('>') - if endpos > -1: - (_, startline, startpos) = ReverseCloseExpression( - clean_lines, linenum, endpos) - if startpos > -1 and startline < linenum: - # Found the matching < on an earlier line, collect all - # pieces up to current line. - line = '' - for i in xrange(startline, linenum + 1): - line += clean_lines.elided[i].strip() - - # Check for non-const references in function parameters. A single '&' may - # found in the following places: - # inside expression: binary & for bitwise AND - # inside expression: unary & for taking the address of something - # inside declarators: reference parameter - # We will exclude the first two cases by checking that we are not inside a - # function body, including one that was just introduced by a trailing '{'. - # TODO(unknown): Doesn't account for 'catch(Exception& e)' [rare]. - if (nesting_state.previous_stack_top and - not (isinstance(nesting_state.previous_stack_top, _ClassInfo) or - isinstance(nesting_state.previous_stack_top, _NamespaceInfo))): - # Not at toplevel, not within a class, and not within a namespace - return - - # Avoid initializer lists. We only need to scan back from the - # current line for something that starts with ':'. - # - # We don't need to check the current line, since the '&' would - # appear inside the second set of parentheses on the current line as - # opposed to the first set. - if linenum > 0: - for i in xrange(linenum - 1, max(0, linenum - 10), -1): - previous_line = clean_lines.elided[i] - if not Search(r'[),]\s*$', previous_line): - break - if Match(r'^\s*:\s+\S', previous_line): - return - - # Avoid preprocessors - if Search(r'\\\s*$', line): - return - - # Avoid constructor initializer lists - if IsInitializerList(clean_lines, linenum): - return - - # We allow non-const references in a few standard places, like functions - # called "swap()" or iostream operators like "<<" or ">>". Do not check - # those function parameters. - # - # We also accept & in static_assert, which looks like a function but - # it's actually a declaration expression. - whitelisted_functions = (r'(?:[sS]wap(?:<\w:+>)?|' - r'operator\s*[<>][<>]|' - r'static_assert|COMPILE_ASSERT' - r')\s*\(') - if Search(whitelisted_functions, line): - return - elif not Search(r'\S+\([^)]*$', line): - # Don't see a whitelisted function on this line. Actually we - # didn't see any function name on this line, so this is likely a - # multi-line parameter list. Try a bit harder to catch this case. - for i in xrange(2): - if (linenum > i and - Search(whitelisted_functions, clean_lines.elided[linenum - i - 1])): - return - - decls = ReplaceAll(r'{[^}]*}', ' ', line) # exclude function body - for parameter in re.findall(_RE_PATTERN_REF_PARAM, decls): - if (not Match(_RE_PATTERN_CONST_REF_PARAM, parameter) and - not Match(_RE_PATTERN_REF_STREAM_PARAM, parameter)): - error(filename, linenum, 'runtime/references', 2, - 'Is this a non-const reference? ' - 'If so, make const or use a pointer: ' + - ReplaceAll(' *<', '<', parameter)) - - -def CheckCasts(filename, clean_lines, linenum, error): - """Various cast related checks. - - Args: - filename: The name of the current file. - clean_lines: A CleansedLines instance containing the file. - linenum: The number of the line to check. - error: The function to call with any errors found. - """ - line = clean_lines.elided[linenum] - - # Check to see if they're using an conversion function cast. - # I just try to capture the most common basic types, though there are more. - # Parameterless conversion functions, such as bool(), are allowed as they are - # probably a member operator declaration or default constructor. - match = Search( - r'(\bnew\s+(?:const\s+)?|\S<\s*(?:const\s+)?)?\b' - r'(int|float|double|bool|char|int32|uint32|int64|uint64)' - r'(\([^)].*)', line) - expecting_function = ExpectingFunctionArgs(clean_lines, linenum) - if match and not expecting_function: - matched_type = match.group(2) - - # matched_new_or_template is used to silence two false positives: - # - New operators - # - Template arguments with function types - # - # For template arguments, we match on types immediately following - # an opening bracket without any spaces. This is a fast way to - # silence the common case where the function type is the first - # template argument. False negative with less-than comparison is - # avoided because those operators are usually followed by a space. - # - # function // bracket + no space = false positive - # value < double(42) // bracket + space = true positive - matched_new_or_template = match.group(1) - - # Avoid arrays by looking for brackets that come after the closing - # parenthesis. - if Match(r'\([^()]+\)\s*\[', match.group(3)): - return - - # Other things to ignore: - # - Function pointers - # - Casts to pointer types - # - Placement new - # - Alias declarations - matched_funcptr = match.group(3) - if (matched_new_or_template is None and - not (matched_funcptr and - (Match(r'\((?:[^() ]+::\s*\*\s*)?[^() ]+\)\s*\(', - matched_funcptr) or - matched_funcptr.startswith('(*)'))) and - not Match(r'\s*using\s+\S+\s*=\s*' + matched_type, line) and - not Search(r'new\(\S+\)\s*' + matched_type, line)): - error(filename, linenum, 'readability/casting', 4, - 'Using deprecated casting style. ' - 'Use static_cast<%s>(...) instead' % - matched_type) - - if not expecting_function: - CheckCStyleCast(filename, clean_lines, linenum, 'static_cast', - r'\((int|float|double|bool|char|u?int(16|32|64))\)', error) - - # This doesn't catch all cases. Consider (const char * const)"hello". - # - # (char *) "foo" should always be a const_cast (reinterpret_cast won't - # compile). - if CheckCStyleCast(filename, clean_lines, linenum, 'const_cast', - r'\((char\s?\*+\s?)\)\s*"', error): - pass - else: - # Check pointer casts for other than string constants - CheckCStyleCast(filename, clean_lines, linenum, 'reinterpret_cast', - r'\((\w+\s?\*+\s?)\)', error) - - # In addition, we look for people taking the address of a cast. This - # is dangerous -- casts can assign to temporaries, so the pointer doesn't - # point where you think. - # - # Some non-identifier character is required before the '&' for the - # expression to be recognized as a cast. These are casts: - # expression = &static_cast(temporary()); - # function(&(int*)(temporary())); - # - # This is not a cast: - # reference_type&(int* function_param); - match = Search( - r'(?:[^\w]&\(([^)*][^)]*)\)[\w(])|' - r'(?:[^\w]&(static|dynamic|down|reinterpret)_cast\b)', line) - if match: - # Try a better error message when the & is bound to something - # dereferenced by the casted pointer, as opposed to the casted - # pointer itself. - parenthesis_error = False - match = Match(r'^(.*&(?:static|dynamic|down|reinterpret)_cast\b)<', line) - if match: - _, y1, x1 = CloseExpression(clean_lines, linenum, len(match.group(1))) - if x1 >= 0 and clean_lines.elided[y1][x1] == '(': - _, y2, x2 = CloseExpression(clean_lines, y1, x1) - if x2 >= 0: - extended_line = clean_lines.elided[y2][x2:] - if y2 < clean_lines.NumLines() - 1: - extended_line += clean_lines.elided[y2 + 1] - if Match(r'\s*(?:->|\[)', extended_line): - parenthesis_error = True - - if parenthesis_error: - error(filename, linenum, 'readability/casting', 4, - ('Are you taking an address of something dereferenced ' - 'from a cast? Wrapping the dereferenced expression in ' - 'parentheses will make the binding more obvious')) - else: - error(filename, linenum, 'runtime/casting', 4, - ('Are you taking an address of a cast? ' - 'This is dangerous: could be a temp var. ' - 'Take the address before doing the cast, rather than after')) - - -def CheckCStyleCast(filename, clean_lines, linenum, cast_type, pattern, error): - """Checks for a C-style cast by looking for the pattern. - - Args: - filename: The name of the current file. - clean_lines: A CleansedLines instance containing the file. - linenum: The number of the line to check. - cast_type: The string for the C++ cast to recommend. This is either - reinterpret_cast, static_cast, or const_cast, depending. - pattern: The regular expression used to find C-style casts. - error: The function to call with any errors found. - - Returns: - True if an error was emitted. - False otherwise. - """ - line = clean_lines.elided[linenum] - match = Search(pattern, line) - if not match: - return False - - # Exclude lines with keywords that tend to look like casts - context = line[0:match.start(1) - 1] - if Match(r'.*\b(?:sizeof|alignof|alignas|[_A-Z][_A-Z0-9]*)\s*$', context): - return False - - # Try expanding current context to see if we one level of - # parentheses inside a macro. - if linenum > 0: - for i in xrange(linenum - 1, max(0, linenum - 5), -1): - context = clean_lines.elided[i] + context - if Match(r'.*\b[_A-Z][_A-Z0-9]*\s*\((?:\([^()]*\)|[^()])*$', context): - return False - - # operator++(int) and operator--(int) - if context.endswith(' operator++') or context.endswith(' operator--'): - return False - - # A single unnamed argument for a function tends to look like old style cast. - # If we see those, don't issue warnings for deprecated casts. - remainder = line[match.end(0):] - if Match(r'^\s*(?:;|const\b|throw\b|final\b|override\b|[=>{),]|->)', - remainder): - return False - - # At this point, all that should be left is actual casts. - error(filename, linenum, 'readability/casting', 4, - 'Using C-style cast. Use %s<%s>(...) instead' % - (cast_type, match.group(1))) - - return True - - -def ExpectingFunctionArgs(clean_lines, linenum): - """Checks whether where function type arguments are expected. - - Args: - clean_lines: A CleansedLines instance containing the file. - linenum: The number of the line to check. - - Returns: - True if the line at 'linenum' is inside something that expects arguments - of function types. - """ - line = clean_lines.elided[linenum] - return (Match(r'^\s*MOCK_(CONST_)?METHOD\d+(_T)?\(', line) or - (linenum >= 2 and - (Match(r'^\s*MOCK_(?:CONST_)?METHOD\d+(?:_T)?\((?:\S+,)?\s*$', - clean_lines.elided[linenum - 1]) or - Match(r'^\s*MOCK_(?:CONST_)?METHOD\d+(?:_T)?\(\s*$', - clean_lines.elided[linenum - 2]) or - Search(r'\bstd::m?function\s*\<\s*$', - clean_lines.elided[linenum - 1])))) - - -_HEADERS_CONTAINING_TEMPLATES = ( - ('', ('deque',)), - ('', ('unary_function', 'binary_function', - 'plus', 'minus', 'multiplies', 'divides', 'modulus', - 'negate', - 'equal_to', 'not_equal_to', 'greater', 'less', - 'greater_equal', 'less_equal', - 'logical_and', 'logical_or', 'logical_not', - 'unary_negate', 'not1', 'binary_negate', 'not2', - 'bind1st', 'bind2nd', - 'pointer_to_unary_function', - 'pointer_to_binary_function', - 'ptr_fun', - 'mem_fun_t', 'mem_fun', 'mem_fun1_t', 'mem_fun1_ref_t', - 'mem_fun_ref_t', - 'const_mem_fun_t', 'const_mem_fun1_t', - 'const_mem_fun_ref_t', 'const_mem_fun1_ref_t', - 'mem_fun_ref', - )), - ('', ('numeric_limits',)), - ('', ('list',)), - ('', ('multimap',)), - ('', ('allocator', 'make_shared', 'make_unique', 'shared_ptr', - 'unique_ptr', 'weak_ptr')), - ('', ('queue', 'priority_queue',)), - ('', ('multiset',)), - ('', ('stack',)), - ('', ('char_traits', 'basic_string',)), - ('', ('tuple',)), - ('', ('unordered_map', 'unordered_multimap')), - ('', ('unordered_set', 'unordered_multiset')), - ('', ('pair',)), - ('', ('vector',)), - - # gcc extensions. - # Note: std::hash is their hash, ::hash is our hash - ('', ('hash_map', 'hash_multimap',)), - ('', ('hash_set', 'hash_multiset',)), - ('', ('slist',)), - ) - -_HEADERS_MAYBE_TEMPLATES = ( - ('', ('copy', 'max', 'min', 'min_element', 'sort', - 'transform', - )), - ('', ('forward', 'make_pair', 'move', 'swap')), - ) - -_RE_PATTERN_STRING = re.compile(r'\bstring\b') - -_re_pattern_headers_maybe_templates = [] -for _header, _templates in _HEADERS_MAYBE_TEMPLATES: - for _template in _templates: - # Match max(..., ...), max(..., ...), but not foo->max, foo.max or - # 'type::max()'. - _re_pattern_headers_maybe_templates.append( - (re.compile(r'[^>.]\b' + _template + r'(<.*?>)?\([^\)]'), - _template, - _header)) -# Match set, but not foo->set, foo.set -_re_pattern_headers_maybe_templates.append( - (re.compile(r'[^>.]\bset\s*\<'), - 'set<>', - '')) -# Match 'map var' and 'std::map(...)', but not 'map(...)'' -_re_pattern_headers_maybe_templates.append( - (re.compile(r'(std\b::\bmap\s*\<)|(^(std\b::\b)map\b\(\s*\<)'), - 'map<>', - '')) - -# Other scripts may reach in and modify this pattern. -_re_pattern_templates = [] -for _header, _templates in _HEADERS_CONTAINING_TEMPLATES: - for _template in _templates: - _re_pattern_templates.append( - (re.compile(r'(\<|\b)' + _template + r'\s*\<'), - _template + '<>', - _header)) - - -def FilesBelongToSameModule(filename_cc, filename_h): - """Check if these two filenames belong to the same module. - - The concept of a 'module' here is a as follows: - foo.h, foo-inl.h, foo.cc, foo_test.cc and foo_unittest.cc belong to the - same 'module' if they are in the same directory. - some/path/public/xyzzy and some/path/internal/xyzzy are also considered - to belong to the same module here. - - If the filename_cc contains a longer path than the filename_h, for example, - '/absolute/path/to/base/sysinfo.cc', and this file would include - 'base/sysinfo.h', this function also produces the prefix needed to open the - header. This is used by the caller of this function to more robustly open the - header file. We don't have access to the real include paths in this context, - so we need this guesswork here. - - Known bugs: tools/base/bar.cc and base/bar.h belong to the same module - according to this implementation. Because of this, this function gives - some false positives. This should be sufficiently rare in practice. - - Args: - filename_cc: is the path for the source (e.g. .cc) file - filename_h: is the path for the header path - - Returns: - Tuple with a bool and a string: - bool: True if filename_cc and filename_h belong to the same module. - string: the additional prefix needed to open the header file. - """ - fileinfo_cc = FileInfo(filename_cc) - if not fileinfo_cc.Extension().lstrip('.') in GetNonHeaderExtensions(): - return (False, '') - - fileinfo_h = FileInfo(filename_h) - if not IsHeaderExtension(fileinfo_h.Extension().lstrip('.')): - return (False, '') - - filename_cc = filename_cc[:-(len(fileinfo_cc.Extension()))] - matched_test_suffix = Search(_TEST_FILE_SUFFIX, fileinfo_cc.BaseName()) - if matched_test_suffix: - filename_cc = filename_cc[:-len(matched_test_suffix.group(1))] - - filename_cc = filename_cc.replace('/public/', '/') - filename_cc = filename_cc.replace('/internal/', '/') - - filename_h = filename_h[:-(len(fileinfo_h.Extension()))] - if filename_h.endswith('-inl'): - filename_h = filename_h[:-len('-inl')] - filename_h = filename_h.replace('/public/', '/') - filename_h = filename_h.replace('/internal/', '/') - - files_belong_to_same_module = filename_cc.endswith(filename_h) - common_path = '' - if files_belong_to_same_module: - common_path = filename_cc[:-len(filename_h)] - return files_belong_to_same_module, common_path - - -def UpdateIncludeState(filename, include_dict, io=codecs): - """Fill up the include_dict with new includes found from the file. - - Args: - filename: the name of the header to read. - include_dict: a dictionary in which the headers are inserted. - io: The io factory to use to read the file. Provided for testability. - - Returns: - True if a header was successfully added. False otherwise. - """ - headerfile = None - try: - headerfile = io.open(filename, 'r', 'utf8', 'replace') - except IOError: - return False - linenum = 0 - for line in headerfile: - linenum += 1 - clean_line = CleanseComments(line) - match = _RE_PATTERN_INCLUDE.search(clean_line) - if match: - include = match.group(2) - include_dict.setdefault(include, linenum) - return True - - -def CheckForIncludeWhatYouUse(filename, clean_lines, include_state, error, - io=codecs): - """Reports for missing stl includes. - - This function will output warnings to make sure you are including the headers - necessary for the stl containers and functions that you use. We only give one - reason to include a header. For example, if you use both equal_to<> and - less<> in a .h file, only one (the latter in the file) of these will be - reported as a reason to include the . - - Args: - filename: The name of the current file. - clean_lines: A CleansedLines instance containing the file. - include_state: An _IncludeState instance. - error: The function to call with any errors found. - io: The IO factory to use to read the header file. Provided for unittest - injection. - """ - required = {} # A map of header name to linenumber and the template entity. - # Example of required: { '': (1219, 'less<>') } - - for linenum in xrange(clean_lines.NumLines()): - line = clean_lines.elided[linenum] - if not line or line[0] == '#': - continue - - # String is special -- it is a non-templatized type in STL. - matched = _RE_PATTERN_STRING.search(line) - if matched: - # Don't warn about strings in non-STL namespaces: - # (We check only the first match per line; good enough.) - prefix = line[:matched.start()] - if prefix.endswith('std::') or not prefix.endswith('::'): - required[''] = (linenum, 'string') - - for pattern, template, header in _re_pattern_headers_maybe_templates: - if pattern.search(line): - required[header] = (linenum, template) - - # The following function is just a speed up, no semantics are changed. - if not '<' in line: # Reduces the cpu time usage by skipping lines. - continue - - for pattern, template, header in _re_pattern_templates: - matched = pattern.search(line) - if matched: - # Don't warn about IWYU in non-STL namespaces: - # (We check only the first match per line; good enough.) - prefix = line[:matched.start()] - if prefix.endswith('std::') or not prefix.endswith('::'): - required[header] = (linenum, template) - - # The policy is that if you #include something in foo.h you don't need to - # include it again in foo.cc. Here, we will look at possible includes. - # Let's flatten the include_state include_list and copy it into a dictionary. - include_dict = dict([item for sublist in include_state.include_list - for item in sublist]) - - # Did we find the header for this file (if any) and successfully load it? - header_found = False - - # Use the absolute path so that matching works properly. - abs_filename = FileInfo(filename).FullName() - - # For Emacs's flymake. - # If cpplint is invoked from Emacs's flymake, a temporary file is generated - # by flymake and that file name might end with '_flymake.cc'. In that case, - # restore original file name here so that the corresponding header file can be - # found. - # e.g. If the file name is 'foo_flymake.cc', we should search for 'foo.h' - # instead of 'foo_flymake.h' - abs_filename = re.sub(r'_flymake\.cc$', '.cc', abs_filename) - - # include_dict is modified during iteration, so we iterate over a copy of - # the keys. - header_keys = list(include_dict.keys()) - for header in header_keys: - (same_module, common_path) = FilesBelongToSameModule(abs_filename, header) - fullpath = common_path + header - if same_module and UpdateIncludeState(fullpath, include_dict, io): - header_found = True - - # If we can't find the header file for a .cc, assume it's because we don't - # know where to look. In that case we'll give up as we're not sure they - # didn't include it in the .h file. - # TODO(unknown): Do a better job of finding .h files so we are confident that - # not having the .h file means there isn't one. - if not header_found: - for extension in GetNonHeaderExtensions(): - if filename.endswith('.' + extension): - return - - # All the lines have been processed, report the errors found. - for required_header_unstripped in sorted(required, key=required.__getitem__): - template = required[required_header_unstripped][1] - if required_header_unstripped.strip('<>"') not in include_dict: - error(filename, required[required_header_unstripped][0], - 'build/include_what_you_use', 4, - 'Add #include ' + required_header_unstripped + ' for ' + template) - - -_RE_PATTERN_EXPLICIT_MAKEPAIR = re.compile(r'\bmake_pair\s*<') - - -def CheckMakePairUsesDeduction(filename, clean_lines, linenum, error): - """Check that make_pair's template arguments are deduced. - - G++ 4.6 in C++11 mode fails badly if make_pair's template arguments are - specified explicitly, and such use isn't intended in any case. - - Args: - filename: The name of the current file. - clean_lines: A CleansedLines instance containing the file. - linenum: The number of the line to check. - error: The function to call with any errors found. - """ - line = clean_lines.elided[linenum] - match = _RE_PATTERN_EXPLICIT_MAKEPAIR.search(line) - if match: - error(filename, linenum, 'build/explicit_make_pair', - 4, # 4 = high confidence - 'For C++11-compatibility, omit template arguments from make_pair' - ' OR use pair directly OR if appropriate, construct a pair directly') - - -def CheckRedundantVirtual(filename, clean_lines, linenum, error): - """Check if line contains a redundant "virtual" function-specifier. - - Args: - filename: The name of the current file. - clean_lines: A CleansedLines instance containing the file. - linenum: The number of the line to check. - error: The function to call with any errors found. - """ - # Look for "virtual" on current line. - line = clean_lines.elided[linenum] - virtual = Match(r'^(.*)(\bvirtual\b)(.*)$', line) - if not virtual: return - - # Ignore "virtual" keywords that are near access-specifiers. These - # are only used in class base-specifier and do not apply to member - # functions. - if (Search(r'\b(public|protected|private)\s+$', virtual.group(1)) or - Match(r'^\s+(public|protected|private)\b', virtual.group(3))): - return - - # Ignore the "virtual" keyword from virtual base classes. Usually - # there is a column on the same line in these cases (virtual base - # classes are rare in google3 because multiple inheritance is rare). - if Match(r'^.*[^:]:[^:].*$', line): return - - # Look for the next opening parenthesis. This is the start of the - # parameter list (possibly on the next line shortly after virtual). - # TODO(unknown): doesn't work if there are virtual functions with - # decltype() or other things that use parentheses, but csearch suggests - # that this is rare. - end_col = -1 - end_line = -1 - start_col = len(virtual.group(2)) - for start_line in xrange(linenum, min(linenum + 3, clean_lines.NumLines())): - line = clean_lines.elided[start_line][start_col:] - parameter_list = Match(r'^([^(]*)\(', line) - if parameter_list: - # Match parentheses to find the end of the parameter list - (_, end_line, end_col) = CloseExpression( - clean_lines, start_line, start_col + len(parameter_list.group(1))) - break - start_col = 0 - - if end_col < 0: - return # Couldn't find end of parameter list, give up - - # Look for "override" or "final" after the parameter list - # (possibly on the next few lines). - for i in xrange(end_line, min(end_line + 3, clean_lines.NumLines())): - line = clean_lines.elided[i][end_col:] - match = Search(r'\b(override|final)\b', line) - if match: - error(filename, linenum, 'readability/inheritance', 4, - ('"virtual" is redundant since function is ' - 'already declared as "%s"' % match.group(1))) - - # Set end_col to check whole lines after we are done with the - # first line. - end_col = 0 - if Search(r'[^\w]\s*$', line): - break - - -def CheckRedundantOverrideOrFinal(filename, clean_lines, linenum, error): - """Check if line contains a redundant "override" or "final" virt-specifier. - - Args: - filename: The name of the current file. - clean_lines: A CleansedLines instance containing the file. - linenum: The number of the line to check. - error: The function to call with any errors found. - """ - # Look for closing parenthesis nearby. We need one to confirm where - # the declarator ends and where the virt-specifier starts to avoid - # false positives. - line = clean_lines.elided[linenum] - declarator_end = line.rfind(')') - if declarator_end >= 0: - fragment = line[declarator_end:] - else: - if linenum > 1 and clean_lines.elided[linenum - 1].rfind(')') >= 0: - fragment = line - else: - return - - # Check that at most one of "override" or "final" is present, not both - if Search(r'\boverride\b', fragment) and Search(r'\bfinal\b', fragment): - error(filename, linenum, 'readability/inheritance', 4, - ('"override" is redundant since function is ' - 'already declared as "final"')) - - - - -# Returns true if we are at a new block, and it is directly -# inside of a namespace. -def IsBlockInNameSpace(nesting_state, is_forward_declaration): - """Checks that the new block is directly in a namespace. - - Args: - nesting_state: The _NestingState object that contains info about our state. - is_forward_declaration: If the class is a forward declared class. - Returns: - Whether or not the new block is directly in a namespace. - """ - if is_forward_declaration: - return len(nesting_state.stack) >= 1 and ( - isinstance(nesting_state.stack[-1], _NamespaceInfo)) - - - return (len(nesting_state.stack) > 1 and - nesting_state.stack[-1].check_namespace_indentation and - isinstance(nesting_state.stack[-2], _NamespaceInfo)) - - -def ShouldCheckNamespaceIndentation(nesting_state, is_namespace_indent_item, - raw_lines_no_comments, linenum): - """This method determines if we should apply our namespace indentation check. - - Args: - nesting_state: The current nesting state. - is_namespace_indent_item: If we just put a new class on the stack, True. - If the top of the stack is not a class, or we did not recently - add the class, False. - raw_lines_no_comments: The lines without the comments. - linenum: The current line number we are processing. - - Returns: - True if we should apply our namespace indentation check. Currently, it - only works for classes and namespaces inside of a namespace. - """ - - is_forward_declaration = IsForwardClassDeclaration(raw_lines_no_comments, - linenum) - - if not (is_namespace_indent_item or is_forward_declaration): - return False - - # If we are in a macro, we do not want to check the namespace indentation. - if IsMacroDefinition(raw_lines_no_comments, linenum): - return False - - return IsBlockInNameSpace(nesting_state, is_forward_declaration) - - -# Call this method if the line is directly inside of a namespace. -# If the line above is blank (excluding comments) or the start of -# an inner namespace, it cannot be indented. -def CheckItemIndentationInNamespace(filename, raw_lines_no_comments, linenum, - error): - line = raw_lines_no_comments[linenum] - if Match(r'^\s+', line): - error(filename, linenum, 'runtime/indentation_namespace', 4, - 'Do not indent within a namespace') - - -def ProcessLine(filename, file_extension, clean_lines, line, - include_state, function_state, nesting_state, error, - extra_check_functions=None): - """Processes a single line in the file. - - Args: - filename: Filename of the file that is being processed. - file_extension: The extension (dot not included) of the file. - clean_lines: An array of strings, each representing a line of the file, - with comments stripped. - line: Number of line being processed. - include_state: An _IncludeState instance in which the headers are inserted. - function_state: A _FunctionState instance which counts function lines, etc. - nesting_state: A NestingState instance which maintains information about - the current stack of nested blocks being parsed. - error: A callable to which errors are reported, which takes 4 arguments: - filename, line number, error level, and message - extra_check_functions: An array of additional check functions that will be - run on each source line. Each function takes 4 - arguments: filename, clean_lines, line, error - """ - raw_lines = clean_lines.raw_lines - ParseNolintSuppressions(filename, raw_lines[line], line, error) - nesting_state.Update(filename, clean_lines, line, error) - CheckForNamespaceIndentation(filename, nesting_state, clean_lines, line, - error) - if nesting_state.InAsmBlock(): return - CheckForFunctionLengths(filename, clean_lines, line, function_state, error) - CheckForMultilineCommentsAndStrings(filename, clean_lines, line, error) - CheckStyle(filename, clean_lines, line, file_extension, nesting_state, error) - CheckLanguage(filename, clean_lines, line, file_extension, include_state, - nesting_state, error) - CheckForNonConstReference(filename, clean_lines, line, nesting_state, error) - CheckForNonStandardConstructs(filename, clean_lines, line, - nesting_state, error) - CheckVlogArguments(filename, clean_lines, line, error) - CheckPosixThreading(filename, clean_lines, line, error) - CheckInvalidIncrement(filename, clean_lines, line, error) - CheckMakePairUsesDeduction(filename, clean_lines, line, error) - CheckRedundantVirtual(filename, clean_lines, line, error) - CheckRedundantOverrideOrFinal(filename, clean_lines, line, error) - if extra_check_functions: - for check_fn in extra_check_functions: - check_fn(filename, clean_lines, line, error) - -def FlagCxx11Features(filename, clean_lines, linenum, error): - """Flag those c++11 features that we only allow in certain places. - - Args: - filename: The name of the current file. - clean_lines: A CleansedLines instance containing the file. - linenum: The number of the line to check. - error: The function to call with any errors found. - """ - line = clean_lines.elided[linenum] - - include = Match(r'\s*#\s*include\s+[<"]([^<"]+)[">]', line) - - # Flag unapproved C++ TR1 headers. - if include and include.group(1).startswith('tr1/'): - error(filename, linenum, 'build/c++tr1', 5, - ('C++ TR1 headers such as <%s> are unapproved.') % include.group(1)) - - # Flag unapproved C++11 headers. - if include and include.group(1) in ('cfenv', - 'condition_variable', - 'fenv.h', - 'future', - 'mutex', - 'thread', - 'chrono', - 'ratio', - 'regex', - 'system_error', - ): - error(filename, linenum, 'build/c++11', 5, - ('<%s> is an unapproved C++11 header.') % include.group(1)) - - # The only place where we need to worry about C++11 keywords and library - # features in preprocessor directives is in macro definitions. - if Match(r'\s*#', line) and not Match(r'\s*#\s*define\b', line): return - - # These are classes and free functions. The classes are always - # mentioned as std::*, but we only catch the free functions if - # they're not found by ADL. They're alphabetical by header. - for top_name in ( - # type_traits - 'alignment_of', - 'aligned_union', - ): - if Search(r'\bstd::%s\b' % top_name, line): - error(filename, linenum, 'build/c++11', 5, - ('std::%s is an unapproved C++11 class or function. Send c-style ' - 'an example of where it would make your code more readable, and ' - 'they may let you use it.') % top_name) - - -def FlagCxx14Features(filename, clean_lines, linenum, error): - """Flag those C++14 features that we restrict. - - Args: - filename: The name of the current file. - clean_lines: A CleansedLines instance containing the file. - linenum: The number of the line to check. - error: The function to call with any errors found. - """ - line = clean_lines.elided[linenum] - - include = Match(r'\s*#\s*include\s+[<"]([^<"]+)[">]', line) - - # Flag unapproved C++14 headers. - if include and include.group(1) in ('scoped_allocator', 'shared_mutex'): - error(filename, linenum, 'build/c++14', 5, - ('<%s> is an unapproved C++14 header.') % include.group(1)) - - -def ProcessFileData(filename, file_extension, lines, error, - extra_check_functions=None): - """Performs lint checks and reports any errors to the given error function. - - Args: - filename: Filename of the file that is being processed. - file_extension: The extension (dot not included) of the file. - lines: An array of strings, each representing a line of the file, with the - last element being empty if the file is terminated with a newline. - error: A callable to which errors are reported, which takes 4 arguments: - filename, line number, error level, and message - extra_check_functions: An array of additional check functions that will be - run on each source line. Each function takes 4 - arguments: filename, clean_lines, line, error - """ - lines = (['// marker so line numbers and indices both start at 1'] + lines + - ['// marker so line numbers end in a known way']) - - include_state = _IncludeState() - function_state = _FunctionState() - nesting_state = NestingState() - - ResetNolintSuppressions() - - CheckForCopyright(filename, lines, error) - ProcessGlobalSuppresions(lines) - RemoveMultiLineComments(filename, lines, error) - clean_lines = CleansedLines(lines) - - if IsHeaderExtension(file_extension): - CheckForHeaderGuard(filename, clean_lines, error) - - for line in xrange(clean_lines.NumLines()): - ProcessLine(filename, file_extension, clean_lines, line, - include_state, function_state, nesting_state, error, - extra_check_functions) - FlagCxx11Features(filename, clean_lines, line, error) - nesting_state.CheckCompletedBlocks(filename, error) - - CheckForIncludeWhatYouUse(filename, clean_lines, include_state, error) - - # Check that the .cc file has included its header if it exists. - if _IsSourceExtension(file_extension): - CheckHeaderFileIncluded(filename, include_state, error) - - # We check here rather than inside ProcessLine so that we see raw - # lines rather than "cleaned" lines. - CheckForBadCharacters(filename, lines, error) - - CheckForNewlineAtEOF(filename, lines, error) - -def ProcessConfigOverrides(filename): - """ Loads the configuration files and processes the config overrides. - - Args: - filename: The name of the file being processed by the linter. - - Returns: - False if the current |filename| should not be processed further. - """ - - abs_filename = os.path.abspath(filename) - cfg_filters = [] - keep_looking = True - while keep_looking: - abs_path, base_name = os.path.split(abs_filename) - if not base_name: - break # Reached the root directory. - - cfg_file = os.path.join(abs_path, "CPPLINT.cfg") - abs_filename = abs_path - if not os.path.isfile(cfg_file): - continue - - try: - with open(cfg_file) as file_handle: - for line in file_handle: - line, _, _ = line.partition('#') # Remove comments. - if not line.strip(): - continue - - name, _, val = line.partition('=') - name = name.strip() - val = val.strip() - if name == 'set noparent': - keep_looking = False - elif name == 'filter': - cfg_filters.append(val) - elif name == 'exclude_files': - # When matching exclude_files pattern, use the base_name of - # the current file name or the directory name we are processing. - # For example, if we are checking for lint errors in /foo/bar/baz.cc - # and we found the .cfg file at /foo/CPPLINT.cfg, then the config - # file's "exclude_files" filter is meant to be checked against "bar" - # and not "baz" nor "bar/baz.cc". - if base_name: - pattern = re.compile(val) - if pattern.match(base_name): - if _cpplint_state.quiet: - # Suppress "Ignoring file" warning when using --quiet. - return False - _cpplint_state.PrintInfo('Ignoring "%s": file excluded by "%s". ' - 'File path component "%s" matches ' - 'pattern "%s"\n' % - (filename, cfg_file, base_name, val)) - return False - elif name == 'linelength': - global _line_length - try: - _line_length = int(val) - except ValueError: - _cpplint_state.PrintError('Line length must be numeric.') - elif name == 'extensions': - ProcessExtensionsOption(val) - elif name == 'root': - global _root - # root directories are specified relative to CPPLINT.cfg dir. - _root = os.path.join(os.path.dirname(cfg_file), val) - elif name == 'headers': - ProcessHppHeadersOption(val) - else: - _cpplint_state.PrintError( - 'Invalid configuration option (%s) in file %s\n' % - (name, cfg_file)) - - except IOError: - _cpplint_state.PrintError( - "Skipping config file '%s': Can't open for reading\n" % cfg_file) - keep_looking = False - - # Apply all the accumulated filters in reverse order (top-level directory - # config options having the least priority). - for cfg_filter in reversed(cfg_filters): - _AddFilters(cfg_filter) - - return True - - -def ProcessFile(filename, vlevel, extra_check_functions=None): - """Does google-lint on a single file. - - Args: - filename: The name of the file to parse. - - vlevel: The level of errors to report. Every error of confidence - >= verbose_level will be reported. 0 is a good default. - - extra_check_functions: An array of additional check functions that will be - run on each source line. Each function takes 4 - arguments: filename, clean_lines, line, error - """ - - _SetVerboseLevel(vlevel) - _BackupFilters() - old_errors = _cpplint_state.error_count - - if not ProcessConfigOverrides(filename): - _RestoreFilters() - return - - lf_lines = [] - crlf_lines = [] - try: - # Support the UNIX convention of using "-" for stdin. Note that - # we are not opening the file with universal newline support - # (which codecs doesn't support anyway), so the resulting lines do - # contain trailing '\r' characters if we are reading a file that - # has CRLF endings. - # If after the split a trailing '\r' is present, it is removed - # below. - if filename == '-': - lines = codecs.StreamReaderWriter(sys.stdin, - codecs.getreader('utf8'), - codecs.getwriter('utf8'), - 'replace').read().split('\n') - else: - lines = codecs.open(filename, 'r', 'utf8', 'replace').read().split('\n') - - # Remove trailing '\r'. - # The -1 accounts for the extra trailing blank line we get from split() - for linenum in range(len(lines) - 1): - if lines[linenum].endswith('\r'): - lines[linenum] = lines[linenum].rstrip('\r') - crlf_lines.append(linenum + 1) - else: - lf_lines.append(linenum + 1) - - except IOError: - _cpplint_state.PrintError( - "Skipping input '%s': Can't open for reading\n" % filename) - _RestoreFilters() - return - - # Note, if no dot is found, this will give the entire filename as the ext. - file_extension = filename[filename.rfind('.') + 1:] - - # When reading from stdin, the extension is unknown, so no cpplint tests - # should rely on the extension. - if filename != '-' and file_extension not in GetAllExtensions(): - _cpplint_state.PrintError('Ignoring %s; not a valid file name ' - '(%s)\n' % (filename, ', '.join(GetAllExtensions()))) - else: - ProcessFileData(filename, file_extension, lines, Error, - extra_check_functions) - - # If end-of-line sequences are a mix of LF and CR-LF, issue - # warnings on the lines with CR. - # - # Don't issue any warnings if all lines are uniformly LF or CR-LF, - # since critique can handle these just fine, and the style guide - # doesn't dictate a particular end of line sequence. - # - # We can't depend on os.linesep to determine what the desired - # end-of-line sequence should be, since that will return the - # server-side end-of-line sequence. - if lf_lines and crlf_lines: - # Warn on every line with CR. An alternative approach might be to - # check whether the file is mostly CRLF or just LF, and warn on the - # minority, we bias toward LF here since most tools prefer LF. - for linenum in crlf_lines: - Error(filename, linenum, 'whitespace/newline', 1, - 'Unexpected \\r (^M) found; better to use only \\n') - - # Suppress printing anything if --quiet was passed unless the error - # count has increased after processing this file. - if not _cpplint_state.quiet or old_errors != _cpplint_state.error_count: - _cpplint_state.PrintInfo('Done processing %s\n' % filename) - _RestoreFilters() - - -def PrintUsage(message): - """Prints a brief usage string and exits, optionally with an error message. - - Args: - message: The optional error message. - """ - sys.stderr.write(_USAGE % (list(GetAllExtensions()), - ','.join(list(GetAllExtensions())), - GetHeaderExtensions(), - ','.join(GetHeaderExtensions()))) - - if message: - sys.exit('\nFATAL ERROR: ' + message) - else: - sys.exit(0) - -def PrintVersion(): - sys.stdout.write('Cpplint fork (https://github.com/cpplint/cpplint)\n') - sys.stdout.write('cpplint ' + __VERSION__ + '\n') - sys.stdout.write('Python ' + sys.version + '\n') - sys.exit(0) - -def PrintCategories(): - """Prints a list of all the error-categories used by error messages. - - These are the categories used to filter messages via --filter. - """ - sys.stderr.write(''.join(' %s\n' % cat for cat in _ERROR_CATEGORIES)) - sys.exit(0) - - -def ParseArguments(args): - """Parses the command line arguments. - - This may set the output format and verbosity level as side-effects. - - Args: - args: The command line arguments: - - Returns: - The list of filenames to lint. - """ - try: - (opts, filenames) = getopt.getopt(args, '', ['help', 'output=', 'verbose=', - 'v=', - 'version', - 'counting=', - 'filter=', - 'root=', - 'repository=', - 'linelength=', - 'extensions=', - 'exclude=', - 'recursive', - 'headers=', - 'quiet']) - except getopt.GetoptError: - PrintUsage('Invalid arguments.') - - verbosity = _VerboseLevel() - output_format = _OutputFormat() - filters = '' - quiet = _Quiet() - counting_style = '' - recursive = False - - for (opt, val) in opts: - if opt == '--help': - PrintUsage(None) - if opt == '--version': - PrintVersion() - elif opt == '--output': - if val not in ('emacs', 'vs7', 'eclipse', 'junit'): - PrintUsage('The only allowed output formats are emacs, vs7, eclipse ' - 'and junit.') - output_format = val - elif opt == '--quiet': - quiet = True - elif opt == '--verbose' or opt == '--v': - verbosity = int(val) - elif opt == '--filter': - filters = val - if not filters: - PrintCategories() - elif opt == '--counting': - if val not in ('total', 'toplevel', 'detailed'): - PrintUsage('Valid counting options are total, toplevel, and detailed') - counting_style = val - elif opt == '--root': - global _root - _root = val - elif opt == '--repository': - global _repository - _repository = val - elif opt == '--linelength': - global _line_length - try: - _line_length = int(val) - except ValueError: - PrintUsage('Line length must be digits.') - elif opt == '--exclude': - global _excludes - if not _excludes: - _excludes = set() - _excludes.update(glob.glob(val)) - elif opt == '--extensions': - ProcessExtensionsOption(val) - elif opt == '--headers': - ProcessHppHeadersOption(val) - elif opt == '--recursive': - recursive = True - - if not filenames: - PrintUsage('No files were specified.') - - if recursive: - filenames = _ExpandDirectories(filenames) - - if _excludes: - filenames = _FilterExcludedFiles(filenames) - - _SetOutputFormat(output_format) - _SetQuiet(quiet) - _SetVerboseLevel(verbosity) - _SetFilters(filters) - _SetCountingStyle(counting_style) - - return filenames - -def _ExpandDirectories(filenames): - """Searches a list of filenames and replaces directories in the list with - all files descending from those directories. Files with extensions not in - the valid extensions list are excluded. - - Args: - filenames: A list of files or directories - - Returns: - A list of all files that are members of filenames or descended from a - directory in filenames - """ - expanded = set() - for filename in filenames: - if not os.path.isdir(filename): - expanded.add(filename) - continue - - for root, _, files in os.walk(filename): - for loopfile in files: - fullname = os.path.join(root, loopfile) - if fullname.startswith('.' + os.path.sep): - fullname = fullname[len('.' + os.path.sep):] - expanded.add(fullname) - - filtered = [] - for filename in expanded: - if os.path.splitext(filename)[1][1:] in GetAllExtensions(): - filtered.append(filename) - - return filtered - -def _FilterExcludedFiles(filenames): - """Filters out files listed in the --exclude command line switch. File paths - in the switch are evaluated relative to the current working directory - """ - exclude_paths = [os.path.abspath(f) for f in _excludes] - return [f for f in filenames if os.path.abspath(f) not in exclude_paths] - -def main(): - filenames = ParseArguments(sys.argv[1:]) - backup_err = sys.stderr - try: - # Change stderr to write with replacement characters so we don't die - # if we try to print something containing non-ASCII characters. - sys.stderr = codecs.StreamReader(sys.stderr, 'replace') - - _cpplint_state.ResetErrorCounts() - for filename in filenames: - ProcessFile(filename, _cpplint_state.verbose_level) - # If --quiet is passed, suppress printing error count unless there are errors. - if not _cpplint_state.quiet or _cpplint_state.error_count > 0: - _cpplint_state.PrintErrorCounts() - - if _cpplint_state.output_format == 'junit': - sys.stderr.write(_cpplint_state.FormatJUnitXML()) - - finally: - sys.stderr = backup_err - - sys.exit(_cpplint_state.error_count > 0) - - -if __name__ == '__main__': - main() diff --git a/build_support/run_clang_format.py b/build_support/run_clang_format.py deleted file mode 100755 index fcc43072d..000000000 --- a/build_support/run_clang_format.py +++ /dev/null @@ -1,127 +0,0 @@ -#!/usr/bin/env python3 -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# -# Modified from the Apache Arrow project for the Terrier project. - -import argparse -import codecs -import difflib -import fnmatch -import os -import subprocess -import sys - - -def check(arguments, source_dir): - formatted_filenames = [] - error = False - for directory, subdirs, filenames in os.walk(source_dir): - fullpaths = (os.path.join(directory, filename) - for filename in filenames) - source_files = [x for x in fullpaths - if x.endswith(".h") or x.endswith(".cpp") or x.endswith(".cc") or x.endswith(".c")] - formatted_filenames.extend( - # Filter out files that match the globs in the globs file - [filename for filename in source_files - if not any((fnmatch.fnmatch(filename, exclude_glob) - for exclude_glob in exclude_globs))]) - - if arguments.fix: - if not arguments.quiet: - # Print out each file on its own line, but run - # clang format once for all of the files - print("\n".join(map(lambda x: "Formatting {}".format(x), - formatted_filenames))) - subprocess.check_call([arguments.clang_format_binary, - "-i"] + formatted_filenames) - else: - for filename in formatted_filenames: - if not arguments.quiet: - print("Checking {}".format(filename)) - # - # Due to some incompatibilities between Python 2 and - # Python 3, there are some specific actions we take here - # to make sure the difflib.unified_diff call works. - # - # In Python 2, the call to subprocess.check_output return - # a 'str' type. In Python 3, however, the call returns a - # 'bytes' type unless the 'encoding' argument is - # specified. Unfortunately, the 'encoding' argument is not - # in the Python 2 API. We could do an if/else here based - # on the version of Python we are running, but it's more - # straightforward to read the file in binary and do utf-8 - # conversion. In Python 2, it's just converting string - # types to unicode types, whereas in Python 3 it's - # converting bytes types to utf-8 encoded str types. This - # approach ensures that the arguments to - # difflib.unified_diff are acceptable string types in both - # Python 2 and Python 3. - with open(filename, "rb") as reader: - # Run clang-format and capture its output - formatted = subprocess.check_output( - [arguments.clang_format_binary, - filename]) - formatted = codecs.decode(formatted, "utf-8") - # Read the original file - original = codecs.decode(reader.read(), "utf-8") - # Run the equivalent of diff -u - diff = list(difflib.unified_diff( - original.splitlines(True), - formatted.splitlines(True), - fromfile=filename, - tofile="{} (after clang format)".format( - filename))) - if diff: - print("{} had clang-format style issues".format(filename)) - # Print out the diff to stderr - error = True - sys.stderr.writelines(diff) - return error - - -if __name__ == "__main__": - parser = argparse.ArgumentParser( - description="Runs clang format on all of the source " - "files. If --fix is specified, and compares the output " - "with the existing file, outputting a unifiied diff if " - "there are any necessary changes") - parser.add_argument("clang_format_binary", - help="Path to the clang-format binary") - parser.add_argument("exclude_globs", - help="Filename containing globs for files " - "that should be excluded from the checks") - parser.add_argument("--source_dirs", - help="Comma-separated root directories of the code") - parser.add_argument("--fix", default=False, - action="store_true", - help="If specified, will re-format the source " - "code instead of comparing the re-formatted " - "output, defaults to %(default)s") - parser.add_argument("--quiet", default=False, - action="store_true", - help="If specified, only print errors") - - args = parser.parse_args() - - had_err = False - exclude_globs = [line.strip() for line in open(args.exclude_globs)] - for source_dir in args.source_dirs.split(','): - if len(source_dir) > 0: - had_err = had_err or check(args, source_dir) - - sys.exit(1 if had_err else 0) diff --git a/build_support/run_clang_tidy.py b/build_support/run_clang_tidy.py deleted file mode 100755 index 913e87126..000000000 --- a/build_support/run_clang_tidy.py +++ /dev/null @@ -1,498 +0,0 @@ -#!/usr/bin/env python3 -# -# ===- run-clang-tidy.py - Parallel clang-tidy runner ---------*- python -*--===# -# -# The LLVM Compiler Infrastructure -# -# This file is distributed under the University of Illinois Open Source -# License. See LICENSE.TXT for details. -# -# Modified from the LLVM project for the Terrier project. -# -# Added `only_diff` cli arg to run clang-tidy on git diff. -# ===------------------------------------------------------------------------===# -# FIXME: Integrate with clang-tidy-diff.py - -""" -Parallel clang-tidy runner -========================== -Runs clang-tidy over all files in a compilation database. Requires clang-tidy -and clang-apply-replacements in $PATH. -Example invocations. -- Run clang-tidy on all files in the current working directory with a default - set of checks and show warnings in the cpp files and all project headers. - run-clang-tidy.py $PWD -- Fix all header guards. - run-clang-tidy.py -fix -checks=-*,llvm-header-guard -- Fix all header guards included from clang-tidy and header guards - for clang-tidy headers. - run-clang-tidy.py -fix -checks=-*,llvm-header-guard extra/clang-tidy \ - -header-filter=extra/clang-tidy -Compilation database setup: -http://clang.llvm.org/docs/HowToSetupToolingForLLVM.html -""" - -from __future__ import division -from __future__ import print_function - -import argparse -import glob -import json -import multiprocessing -import os -import pprint # TERRIER: we want to print out formatted lists of files -import re -import shutil -import subprocess -import sys -import tempfile -import threading -import traceback - -# import yaml # TERRIER: not necessary if we don't want automatic fixes - -from run_clang_tidy_extra import CheckConfig - -is_py2 = sys.version[0] == "2" - -if is_py2: - import Queue as queue -else: - import queue as queue - - -def find_compilation_database(path): - """Adjusts the directory until a compilation database is found.""" - result = "./" - while not os.path.isfile(os.path.join(result, path)): - if os.path.realpath(result) == "/": - print("Error: could not find compilation database.") - sys.exit(1) - result += "../" - return os.path.realpath(result) - - -def make_absolute(f, directory): - if os.path.isabs(f): - return f - return os.path.normpath(os.path.join(directory, f)) - - -def supports_color(): - """ - Modified from https://github.com/django/django/blob/main/django/core/management/color.py - Return True if the running system's terminal supports color, - and False otherwise. - """ - - # isatty is not always implemented, #6223. - is_a_tty = hasattr(sys.stdout, "isatty") and sys.stdout.isatty() - - return is_a_tty and ( - sys.platform != "win32" - or "ANSICON" in os.environ - or - # Windows Terminal supports VT codes. - "WT_SESSION" in os.environ - or - # Microsoft Visual Studio Code's built-in terminal supports colors. - os.environ.get("TERM_PROGRAM") == "vscode" - ) - - -def get_tidy_invocation( - f, - clang_tidy_binary, - checks, - tmpdir, - build_path, - header_filter, - extra_arg, - extra_arg_before, - quiet, - config, -): - """Gets a command line for clang-tidy.""" - start = [clang_tidy_binary] - - if supports_color(): - start.append("--use-color") - - if header_filter is not None: - start.append("-header-filter=" + header_filter) - else: - # Show warnings in all in-project headers by default. - # start.append('-header-filter=^' + build_path + '/.*') - # TERRIER: we have our .clang-tidy file - pass - if checks: - start.append("-checks=" + checks) - if tmpdir is not None: - start.append("-export-fixes") - # Get a temporary file. We immediately close the handle so clang-tidy can - # overwrite it. - (handle, name) = tempfile.mkstemp(suffix=".yaml", dir=tmpdir) - os.close(handle) - start.append(name) - for arg in extra_arg: - start.append("-extra-arg=%s" % arg) - for arg in extra_arg_before: - start.append("-extra-arg-before=%s" % arg) - start.append("-p=" + build_path) - if quiet: - start.append("-quiet") - if config: - start.append("-config=" + config) - start.append(f) - return start - - -def merge_replacement_files(tmpdir, mergefile): - """Merge all replacement files in a directory into a single file""" - # The fixes suggested by clang-tidy >= 4.0.0 are given under - # the top level key 'Diagnostics' in the output yaml files - mergekey = "Diagnostics" - merged = [] - for replacefile in glob.iglob(os.path.join(tmpdir, "*.yaml")): - content = yaml.safe_load(open(replacefile, "r")) - if not content: - continue # Skip empty files. - merged.extend(content.get(mergekey, [])) - - if merged: - # MainSourceFile: The key is required by the definition inside - # include/clang/Tooling/ReplacementsYaml.h, but the value - # is actually never used inside clang-apply-replacements, - # so we set it to '' here. - output = {"MainSourceFile": "", mergekey: merged} - with open(mergefile, "w") as out: - yaml.safe_dump(output, out) - else: - # Empty the file: - open(mergefile, "w").close() - - -def check_clang_apply_replacements_binary(args): - """Checks if invoking supplied clang-apply-replacements binary works.""" - try: - subprocess.check_call([args.clang_apply_replacements_binary, "--version"]) - except: - print( - "Unable to run clang-apply-replacements. Is clang-apply-replacements " - "binary correctly specified?", - file=sys.stderr, - ) - traceback.print_exc() - sys.exit(1) - - -def apply_fixes(args, tmpdir): - """Calls clang-apply-fixes on a given directory.""" - invocation = [args.clang_apply_replacements_binary] - if args.format: - invocation.append("-format") - if args.style: - invocation.append("-style=" + args.style) - invocation.append(tmpdir) - subprocess.call(invocation) - - -def run_tidy(args, tmpdir, build_path, queue, lock, failed_files): - """Takes filenames out of queue and runs clang-tidy on them.""" - while True: - name = queue.get() - print("Checking: {}".format(name)) - sys.stdout.flush() - invocation = get_tidy_invocation( - name, - args.clang_tidy_binary, - args.checks, - tmpdir, - build_path, - args.header_filter, - args.extra_arg, - args.extra_arg_before, - args.quiet, - args.config, - ) - cc = CheckConfig() - # name is the full path of the file for clang-tidy to check - if cc.should_skip(name): - queue.task_done() - continue - - proc = subprocess.Popen( - invocation, stdout=subprocess.PIPE, stderr=subprocess.PIPE - ) - output, err = proc.communicate() - if proc.returncode != 0: - failed_files.append(name) - # TERRIER: we write our own printing logic - # with lock: - # sys.stdout.write(' '.join(invocation) + '\n' + output + '\n') - # if err > 0: - # sys.stderr.write(err + '\n') - # In particular, we only want important lines: - with lock: - output = output.decode("utf-8") if output is not None else None - err = err.decode("utf-8") if output is not None else None - # unfortunately, our error messages are actually on STDOUT - # STDERR tells how many warnings are generated, - # but this includes non-user-code warnings, so it is useless... - if output: - sys.stdout.write("\n") - sys.stdout.write(output) - queue.task_done() - - -def main(): - parser = argparse.ArgumentParser( - description="Runs clang-tidy over all files " - "in a compilation database. Requires " - "clang-tidy and clang-apply-replacements in " - "$PATH." - ) - parser.add_argument( - "-clang-tidy-binary", - metavar="PATH", - default="clang-tidy", - help="path to clang-tidy binary", - ) - parser.add_argument( - "-clang-apply-replacements-binary", - metavar="PATH", - default="clang-apply-replacements", - help="path to clang-apply-replacements binary", - ) - parser.add_argument( - "-checks", - default=None, - help="checks filter, when not specified, use clang-tidy " "default", - ) - parser.add_argument( - "-config", - default=None, - help="Specifies a configuration in YAML/JSON format: " - " -config=\"{Checks: '*', " - " CheckOptions: [{key: x, " - ' value: y}]}" ' - "When the value is empty, clang-tidy will " - "attempt to find a file named .clang-tidy for " - "each source file in its parent directories.", - ) - parser.add_argument( - "-header-filter", - default=None, - help="regular expression matching the names of the " - "headers to output diagnostics from. Diagnostics from " - "the main file of each translation unit are always " - "displayed.", - ) - parser.add_argument( - "-export-fixes", - metavar="filename", - dest="export_fixes", - help="Create a yaml file to store suggested fixes in, " - "which can be applied with clang-apply-replacements.", - ) - parser.add_argument( - "-j", - type=int, - default=0, - help="number of tidy instances to be run in parallel.", - ) - parser.add_argument( - "files", nargs="*", default=[".*"], help="files to be processed (regex on path)" - ) - parser.add_argument("-fix", action="store_true", help="apply fix-its") - parser.add_argument( - "-format", action="store_true", help="Reformat code " "after applying fixes" - ) - parser.add_argument( - "-style", - default="file", - help="The style of reformat " "code after applying fixes", - ) - parser.add_argument( - "-p", dest="build_path", help="Path used to read a compile command database." - ) - parser.add_argument( - "-extra-arg", - dest="extra_arg", - action="append", - default=[], - help="Additional argument to append to the compiler " "command line.", - ) - parser.add_argument( - "-extra-arg-before", - dest="extra_arg_before", - action="append", - default=[], - help="Additional argument to prepend to the compiler " "command line.", - ) - parser.add_argument( - "-quiet", action="store_true", help="Run clang-tidy in quiet mode" - ) - parser.add_argument( - "-only-diff", - action="store_true", - help="Only run clang-tidy on diff file to master branch", - ) - args = parser.parse_args() - - db_path = "compile_commands.json" - - if args.build_path is not None: - build_path = args.build_path - else: - # Find our database - build_path = find_compilation_database(db_path) - - try: - invocation = [args.clang_tidy_binary, "-list-checks"] - invocation.append("-p=" + build_path) - if args.checks: - invocation.append("-checks=" + args.checks) - invocation.append("-") - subprocess.check_call(invocation) - except: - print("Unable to run clang-tidy.", file=sys.stderr) - sys.exit(1) - - # Load the database and extract all files. - database = json.load(open(os.path.join(build_path, db_path))) - files = [make_absolute(entry["file"], entry["directory"]) for entry in database] - - # Running clang-tidy in the whole project is slow. Therefore, we added - # support for running clang-tidy on git diff. When `only_diff` is - # specified in the command line, we only check files modified compared - # with origin/master, so as to speed up clang-tidy check. - # - # This functionality is set as CMake target `check-clang-tidy-diff`. - # You can use `make check-clang-tidy-diff` to do a fast clang-tidy - # check. - if args.only_diff: - # Get the path of the repo, e.g. /Users/terrier/bustub - git_repo = subprocess.run( - ["git", "rev-parse", "--show-toplevel"], capture_output=True - ) - git_repo_path = git_repo.stdout.decode("utf-8").strip() - # Get all files changed compared with origin/master - result = subprocess.run( - ["git", "--no-pager", "diff", "--name-only", "origin/master"], - capture_output=True, - ) - git_changed_file_list = list( - map( - lambda x: make_absolute(x, git_repo_path), - result.stdout.decode("utf-8").strip().split("\n"), - ) - ) - git_changed_file_set = set(git_changed_file_list) - # Only retain files that exists in git diff - files = list(filter(lambda x: x in git_changed_file_set, files)) - - max_task = args.j - if max_task == 0: - max_task = multiprocessing.cpu_count() - - tmpdir = None - if args.fix or args.export_fixes: - check_clang_apply_replacements_binary(args) - tmpdir = tempfile.mkdtemp() - - # Build up a big regexy filter from all command line arguments. - file_name_re = re.compile("|".join(args.files)) - - return_code = 0 - try: - # Spin up a bunch of tidy-launching threads. - task_queue = queue.Queue(max_task) - # List of files with a non-zero return code. - failed_files = [] - lock = threading.Lock() - for _ in range(max_task): - t = threading.Thread( - target=run_tidy, - args=(args, tmpdir, build_path, task_queue, lock, failed_files), - ) - t.daemon = True - t.start() - - def update_progress(current_file, num_files): - pct = int(current_file / num_files * 100) - if current_file == num_files or pct % max(2, num_files // 10) == 0: - stars = pct // 10 - spaces = 10 - pct // 10 - print( - "\rProgress: [{}{}] ({}% / File {} of {})".format( - "x" * stars, " " * spaces, pct, current_file, num_files - ), - end="", - ) - sys.stdout.flush() - if current_file == num_files: - print() - - # Fill the queue with files. - for i, name in enumerate(files): - if file_name_re.search(name): - if name.endswith("tools/backtrace.cpp"): - continue - put_file = False - while not put_file: - try: - task_queue.put(name, block=True, timeout=300) - put_file = True - # update_progress(i, len(files)) - except queue.Full: - print("Still waiting to put files into clang-tidy queue.") - sys.stdout.flush() - - # Wait for all threads to be done. - task_queue.join() - # update_progress(100, 100) - if len(failed_files): - return_code = 1 - # TERRIER: We want to see the failed files - print("The files that failed were:") - print(pprint.pformat(failed_files)) - print( - "Note that a failing .h file will fail all the .cpp files that include it.\n" - ) - - except KeyboardInterrupt: - # This is a sad hack. Unfortunately subprocess goes - # bonkers with ctrl-c and we start forking merrily. - print("\nCtrl-C detected, goodbye.") - if tmpdir: - shutil.rmtree(tmpdir) - os.kill(0, 9) - - if args.export_fixes: - print("Writing fixes to " + args.export_fixes + " ...") - try: - merge_replacement_files(tmpdir, args.export_fixes) - except: - print("Error exporting fixes.\n", file=sys.stderr) - traceback.print_exc() - return_code = 1 - - if args.fix: - print("Applying fixes ...") - try: - apply_fixes(args, tmpdir) - except: - print("Error applying fixes.\n", file=sys.stderr) - traceback.print_exc() - return_code = 1 - - if tmpdir: - shutil.rmtree(tmpdir) - print("") - sys.stdout.flush() - sys.exit(return_code) - - -if __name__ == "__main__": - main() diff --git a/build_support/run_clang_tidy_extra.py b/build_support/run_clang_tidy_extra.py deleted file mode 100644 index a3eb7bb8d..000000000 --- a/build_support/run_clang_tidy_extra.py +++ /dev/null @@ -1,46 +0,0 @@ -#!/usr/bin/env python3 - -""" -A helper class, to suppress execution of clang-tidy. - -In clang-tidy-6.0, if the clang-tidy configuration file suppresses ALL checks, -(e.g. via a .clang-tidy file), clang-tidy will print usage information and -exit with a return code of 0. Harmless but verbose. In later versions of -clang-tidy the return code becomes 1, making this a bigger problem. - -This helper addresses the problem by suppressing execution according to -the configuration in this file. -""" - -import re - -class CheckConfig(object): - """ Check paths against the built-in config """ - - def __init__(self): - self._init_config() - # debug prints - self.debug = False - return - - def _init_config(self): - """ Any path matching one of the ignore_pats regular expressions, - denotes that we do NOT want to run clang-tidy on that item. - """ - self.ignore_pats = [".*/third_party/.*", ] - return - - def should_skip(self, path): - """ Should execution of clang-tidy be skipped? - path - to check, against the configuration. - Typically the full path. - returns - False if we want to run clang-tidy - True of we want to skip execution on this item - """ - for pat in self.ignore_pats: - if re.match(pat, path): - if self.debug: - print("match pat: {}, {} => don't run".format(pat, path)) - return True - return False - diff --git a/src/client.cc b/src/client.cc index fbb358779..42971ba88 100644 --- a/src/client.cc +++ b/src/client.cc @@ -133,6 +133,10 @@ void CmdRes::SetRes(CmdRes::CmdRet _ret, const std::string& content) { case kInvalidCursor: AppendStringRaw("-ERR invalid cursor"); break; + case kmultikey: + AppendStringRaw("-WRONGTYPE Operation against a key holding the wrong kind of value"); + AppendStringRaw(CRLF); + break; case kWrongLeader: AppendStringRaw("-ERR wrong leader"); AppendStringRaw(content); diff --git a/src/client.h b/src/client.h index 7f9eb9823..881ffe562 100644 --- a/src/client.h +++ b/src/client.h @@ -48,6 +48,7 @@ class CmdRes { kErrOther, KIncrByOverFlow, kInvalidCursor, + kmultikey, kWrongLeader, }; @@ -107,6 +108,9 @@ enum class ClientState { kClosed, }; +constexpr const char* ErrTypeMessage = + "Invalid argument: WRONGTYPE Operation against a key holding the wrong kind of value"; + class DB; struct PSlaveInfo; diff --git a/src/cmd_hash.cc b/src/cmd_hash.cc index 60f2e3b01..bd041d227 100644 --- a/src/cmd_hash.cc +++ b/src/cmd_hash.cc @@ -39,13 +39,15 @@ void HSetCmd::DoCmd(PClient* client) { s = PSTORE.GetBackend(client->GetCurrentDB())->GetStorage()->HSet(client->Key(), field, value, &temp); if (s.ok()) { ret += temp; + } else if (s.ToString() == ErrTypeMessage) { + client->SetRes(CmdRes::kmultikey); + return; } else { // FIXME(century): need txn, if bw crashes, it should rollback client->SetRes(CmdRes::kErrOther); return; } } - client->AppendInteger(ret); } @@ -63,6 +65,8 @@ void HGetCmd::DoCmd(PClient* client) { storage::Status s = PSTORE.GetBackend(client->GetCurrentDB())->GetStorage()->HGet(client->Key(), field, &value); if (s.ok()) { client->AppendString(value); + } else if (s.ToString() == ErrTypeMessage) { + client->SetRes(CmdRes::kmultikey); } else if (s.IsNotFound()) { client->AppendString(""); } else { @@ -82,6 +86,10 @@ void HDelCmd::DoCmd(PClient* client) { int32_t res{}; std::vector fields(client->argv_.begin() + 2, client->argv_.end()); auto s = PSTORE.GetBackend(client->GetCurrentDB())->GetStorage()->HDel(client->Key(), fields, &res); + if (s.ToString() == ErrTypeMessage) { + client->SetRes(CmdRes::kmultikey); + return; + } if (!s.ok() && !s.IsNotFound()) { client->SetRes(CmdRes::kErrOther, s.ToString()); return; @@ -106,6 +114,8 @@ void HMSetCmd::DoCmd(PClient* client) { storage::Status s = PSTORE.GetBackend(client->GetCurrentDB())->GetStorage()->HMSet(client->Key(), client->Fvs()); if (s.ok()) { client->SetRes(CmdRes::kOK); + } else if (s.ToString() == ErrTypeMessage) { + client->SetRes(CmdRes::kmultikey); } else { client->SetRes(CmdRes::kErrOther, s.ToString()); } @@ -134,6 +144,8 @@ void HMGetCmd::DoCmd(PClient* client) { client->AppendString(""); } } + } else if (s.ToString() == ErrTypeMessage) { + client->SetRes(CmdRes::kmultikey); } else { client->SetRes(CmdRes::kErrOther, s.ToString()); } @@ -184,6 +196,8 @@ void HGetAllCmd::DoCmd(PClient* client) { if (s.ok() || s.IsNotFound()) { client->AppendArrayLen(total_fv * 2); client->AppendStringRaw(raw); + } else if (s.ToString() == ErrTypeMessage) { + client->SetRes(CmdRes::kmultikey); } else { client->SetRes(CmdRes::kErrOther, s.ToString()); } @@ -208,6 +222,8 @@ void HKeysCmd::DoCmd(PClient* client) { } // update fields client->Fields() = std::move(fields); + } else if (s.ToString() == ErrTypeMessage) { + client->SetRes(CmdRes::kmultikey); } else { client->SetRes(CmdRes::kErrOther, s.ToString()); } @@ -226,6 +242,8 @@ void HLenCmd::DoCmd(PClient* client) { auto s = PSTORE.GetBackend(client->GetCurrentDB())->GetStorage()->HLen(client->Key(), &len); if (s.ok() || s.IsNotFound()) { client->AppendInteger(len); + } else if (s.ToString() == ErrTypeMessage) { + client->SetRes(CmdRes::kmultikey); } else { client->SetRes(CmdRes::kErrOther, "something wrong in hlen"); } @@ -244,6 +262,8 @@ void HStrLenCmd::DoCmd(PClient* client) { auto s = PSTORE.GetBackend(client->GetCurrentDB())->GetStorage()->HStrlen(client->Key(), client->argv_[2], &len); if (s.ok() || s.IsNotFound()) { client->AppendInteger(len); + } else if (s.ToString() == ErrTypeMessage) { + client->SetRes(CmdRes::kmultikey); } else { client->SetRes(CmdRes::kErrOther, "something wrong in hstrlen"); } @@ -295,6 +315,10 @@ void HScanCmd::DoCmd(PClient* client) { auto status = PSTORE.GetBackend(client->GetCurrentDB()) ->GetStorage() ->HScan(client->Key(), cursor, pattern, count, &fvs, &next_cursor); + if (status.ToString() == ErrTypeMessage) { + client->SetRes(CmdRes::kmultikey); + return; + } if (!status.ok() && !status.IsNotFound()) { client->SetRes(CmdRes::kErrOther, status.ToString()); return; @@ -323,6 +347,8 @@ void HValsCmd::DoCmd(PClient* client) { storage::Status s = PSTORE.GetBackend(client->GetCurrentDB())->GetStorage()->HVals(client->Key(), &valueVec); if (s.ok() || s.IsNotFound()) { client->AppendStringVector(valueVec); + } else if (s.ToString() == ErrTypeMessage) { + client->SetRes(CmdRes::kmultikey); } else { client->SetRes(CmdRes::kErrOther, "hvals cmd error"); } @@ -353,6 +379,8 @@ void HIncrbyFloatCmd::DoCmd(PClient* client) { ->HIncrbyfloat(client->Key(), client->argv_[2], client->argv_[3], &newValue); if (s.ok() || s.IsNotFound()) { client->AppendString(newValue); + } else if (s.ToString() == ErrTypeMessage) { + client->SetRes(CmdRes::kmultikey); } else { client->SetRes(CmdRes::kErrOther, "hvals cmd error"); } @@ -374,6 +402,8 @@ void HSetNXCmd::DoCmd(PClient* client) { ->HSetnx(client->Key(), client->argv_[2], client->argv_[3], &temp); if (s.ok()) { client->AppendInteger(temp); + } else if (s.ToString() == ErrTypeMessage) { + client->SetRes(CmdRes::kmultikey); } else { client->SetRes(CmdRes::kSyntaxErr, "hsetnx cmd error"); } @@ -400,6 +430,8 @@ void HIncrbyCmd::DoCmd(PClient* client) { PSTORE.GetBackend(client->GetCurrentDB())->GetStorage()->HIncrby(client->Key(), client->argv_[2], int_by, &temp); if (s.ok() || s.IsNotFound()) { client->AppendInteger(temp); + } else if (s.ToString() == ErrTypeMessage) { + client->SetRes(CmdRes::kmultikey); } else { client->SetRes(CmdRes::kErrOther, "hincrby cmd error"); } @@ -440,6 +472,10 @@ void HRandFieldCmd::DoCmd(PClient* client) { // execute command std::vector res; auto s = PSTORE.GetBackend(client->GetCurrentDB())->GetStorage()->HRandField(client->Key(), count, with_values, &res); + if (s.ToString() == ErrTypeMessage) { + client->SetRes(CmdRes::kmultikey); + return; + } if (s.IsNotFound()) { client->AppendString(""); return; @@ -473,6 +509,10 @@ void HExistsCmd::DoCmd(PClient* client) { // execute command std::vector res; auto s = PSTORE.GetBackend(client->GetCurrentDB())->GetStorage()->HExists(client->Key(), field); + if (s.ToString() == ErrTypeMessage) { + client->SetRes(CmdRes::kmultikey); + return; + } if (!s.ok() && !s.IsNotFound()) { return client->SetRes(CmdRes::kErrOther, s.ToString()); } diff --git a/src/cmd_keys.cc b/src/cmd_keys.cc index d1e40e3ca..795cef228 100644 --- a/src/cmd_keys.cc +++ b/src/cmd_keys.cc @@ -63,10 +63,10 @@ bool TypeCmd::DoInitial(PClient* client) { } void TypeCmd::DoCmd(PClient* client) { - std::vector types(1); - rocksdb::Status s = PSTORE.GetBackend(client->GetCurrentDB())->GetStorage()->GetType(client->Key(), true, types); + std::string types; + rocksdb::Status s = PSTORE.GetBackend(client->GetCurrentDB())->GetStorage()->GetType(client->Key(), types); if (s.ok()) { - client->AppendContent("+" + types[0]); + client->AppendContent("+" + types); } else { client->SetRes(CmdRes::kErrOther, s.ToString()); } @@ -103,28 +103,10 @@ bool TtlCmd::DoInitial(PClient* client) { } void TtlCmd::DoCmd(PClient* client) { - std::map type_timestamp; + int64_t type_timestamp; std::map type_status; - type_timestamp = PSTORE.GetBackend(client->GetCurrentDB())->GetStorage()->TTL(client->Key(), &type_status); - for (const auto& item : type_timestamp) { - if (item.second == -3) { - client->SetRes(CmdRes::kErrOther, "ttl internal error"); - return; - } - } - if (type_timestamp[storage::kStrings] != -2) { - client->AppendInteger(type_timestamp[storage::kStrings]); - } else if (type_timestamp[storage::kHashes] != -2) { - client->AppendInteger(type_timestamp[storage::kHashes]); - } else if (type_timestamp[storage::kLists] != -2) { - client->AppendInteger(type_timestamp[storage::kLists]); - } else if (type_timestamp[storage::kZSets] != -2) { - client->AppendInteger(type_timestamp[storage::kZSets]); - } else if (type_timestamp[storage::kSets] != -2) { - client->AppendInteger(type_timestamp[storage::kSets]); - } else { - client->AppendInteger(-2); - } + type_timestamp = PSTORE.GetBackend(client->GetCurrentDB())->GetStorage()->TTL(client->Key()); + client->AppendInteger(type_timestamp); } PExpireCmd::PExpireCmd(const std::string& name, int16_t arity) @@ -203,20 +185,8 @@ bool PersistCmd::DoInitial(PClient* client) { } void PersistCmd::DoCmd(PClient* client) { - std::map type_status; - auto res = PSTORE.GetBackend(client->GetCurrentDB())->GetStorage()->Persist(client->Key(), &type_status); - if (res != -1) { - client->AppendInteger(res); - } else { - std::string cnt; - for (auto const& s : type_status) { - cnt.append(storage::DataTypeToString[s.first]); - cnt.append(" - "); - cnt.append(s.second.ToString()); - cnt.append(";"); - } - client->SetRes(CmdRes::kErrOther, cnt); - } + auto res = PSTORE.GetBackend(client->GetCurrentDB())->GetStorage()->Persist(client->Key()); + client->AppendInteger(res); } KeysCmd::KeysCmd(const std::string& name, int16_t arity) @@ -250,49 +220,8 @@ bool PttlCmd::DoInitial(PClient* client) { // like Blackwidow , Floyd still possible has same key in different data structure void PttlCmd::DoCmd(PClient* client) { - std::map type_status; - auto type_timestamp = PSTORE.GetBackend(client->GetCurrentDB())->GetStorage()->TTL(client->Key(), &type_status); - for (const auto& item : type_timestamp) { - // mean operation exception errors happen in database - if (item.second == -3) { - client->SetRes(CmdRes::kErrOther, "ttl internal error"); - return; - } - } - if (type_timestamp[storage::kStrings] != -2) { - if (type_timestamp[storage::kStrings] == -1) { - client->AppendInteger(-1); - } else { - client->AppendInteger(type_timestamp[storage::kStrings] * 1000); - } - } else if (type_timestamp[storage::kHashes] != -2) { - if (type_timestamp[storage::kHashes] == -1) { - client->AppendInteger(-1); - } else { - client->AppendInteger(type_timestamp[storage::kHashes] * 1000); - } - } else if (type_timestamp[storage::kLists] != -2) { - if (type_timestamp[storage::kLists] == -1) { - client->AppendInteger(-1); - } else { - client->AppendInteger(type_timestamp[storage::kLists] * 1000); - } - } else if (type_timestamp[storage::kSets] != -2) { - if (type_timestamp[storage::kSets] == -1) { - client->AppendInteger(-1); - } else { - client->AppendInteger(type_timestamp[storage::kSets] * 1000); - } - } else if (type_timestamp[storage::kZSets] != -2) { - if (type_timestamp[storage::kZSets] == -1) { - client->AppendInteger(-1); - } else { - client->AppendInteger(type_timestamp[storage::kZSets] * 1000); - } - } else { - // this key not exist - client->AppendInteger(-2); - } + auto timestamp = PSTORE.GetBackend(client->GetCurrentDB())->GetStorage()->TTL(client->Key()); + client->AppendInteger(timestamp); } RenameCmd::RenameCmd(const std::string& name, int16_t arity) diff --git a/src/cmd_list.cc b/src/cmd_list.cc index 53e2d9a7f..0d2da74b5 100644 --- a/src/cmd_list.cc +++ b/src/cmd_list.cc @@ -25,6 +25,8 @@ void LPushCmd::DoCmd(PClient* client) { PSTORE.GetBackend(client->GetCurrentDB())->GetStorage()->LPush(client->Key(), list_values, &reply_num); if (s.ok()) { client->AppendInteger(reply_num); + } else if (s.ToString() == ErrTypeMessage) { + client->SetRes(CmdRes::kmultikey); } else { client->SetRes(CmdRes::kSyntaxErr, "lpush cmd error"); } @@ -45,6 +47,8 @@ void LPushxCmd::DoCmd(PClient* client) { PSTORE.GetBackend(client->GetCurrentDB())->GetStorage()->LPushx(client->Key(), list_values, &reply_num); if (s.ok() || s.IsNotFound()) { client->AppendInteger(reply_num); + } else if (s.ToString() == ErrTypeMessage) { + client->SetRes(CmdRes::kmultikey); } else { client->SetRes(CmdRes::kErrOther, s.ToString()); } @@ -90,6 +94,8 @@ void RPushCmd::DoCmd(PClient* client) { PSTORE.GetBackend(client->GetCurrentDB())->GetStorage()->RPush(client->Key(), list_values, &reply_num); if (s.ok()) { client->AppendInteger(reply_num); + } else if (s.ToString() == ErrTypeMessage) { + client->SetRes(CmdRes::kmultikey); } else { client->SetRes(CmdRes::kSyntaxErr, "rpush cmd error"); } @@ -110,6 +116,8 @@ void RPushxCmd::DoCmd(PClient* client) { PSTORE.GetBackend(client->GetCurrentDB())->GetStorage()->RPushx(client->Key(), list_values, &reply_num); if (s.ok() || s.IsNotFound()) { client->AppendInteger(reply_num); + } else if (s.ToString() == ErrTypeMessage) { + client->SetRes(CmdRes::kmultikey); } else { client->SetRes(CmdRes::kErrOther, s.ToString()); } @@ -128,6 +136,8 @@ void LPopCmd::DoCmd(PClient* client) { storage::Status s = PSTORE.GetBackend(client->GetCurrentDB())->GetStorage()->LPop(client->Key(), 1, &elements); if (s.ok()) { client->AppendString(elements[0]); + } else if (s.ToString() == ErrTypeMessage) { + client->SetRes(CmdRes::kmultikey); } else if (s.IsNotFound()) { client->AppendStringLen(-1); } else { @@ -148,6 +158,8 @@ void RPopCmd::DoCmd(PClient* client) { storage::Status s = PSTORE.GetBackend(client->GetCurrentDB())->GetStorage()->RPop(client->Key(), 1, &elements); if (s.ok()) { client->AppendString(elements[0]); + } else if (s.ToString() == ErrTypeMessage) { + client->SetRes(CmdRes::kmultikey); } else if (s.IsNotFound()) { client->AppendStringLen(-1); } else { @@ -173,6 +185,10 @@ void LRangeCmd::DoCmd(PClient* client) { } storage::Status s = PSTORE.GetBackend(client->GetCurrentDB())->GetStorage()->LRange(client->Key(), start_index, end_index, &ret); + if (s.ToString() == ErrTypeMessage) { + client->SetRes(CmdRes::kmultikey); + return; + } if (!s.ok() && !s.IsNotFound()) { client->SetRes(CmdRes::kSyntaxErr, "lrange cmd error"); return; @@ -201,6 +217,8 @@ void LRemCmd::DoCmd(PClient* client) { PSTORE.GetBackend(client->GetCurrentDB())->GetStorage()->LRem(client->Key(), freq_, client->argv_[3], &reply_num); if (s.ok() || s.IsNotFound()) { client->AppendInteger(reply_num); + } else if (s.ToString() == ErrTypeMessage) { + client->SetRes(CmdRes::kmultikey); } else { client->SetRes(CmdRes::kErrOther, "lrem cmd error"); } @@ -226,6 +244,8 @@ void LTrimCmd::DoCmd(PClient* client) { PSTORE.GetBackend(client->GetCurrentDB())->GetStorage()->LTrim(client->Key(), start_index, end_index); if (s.ok() || s.IsNotFound()) { client->SetRes(CmdRes::kOK); + } else if (s.ToString() == ErrTypeMessage) { + client->SetRes(CmdRes::kmultikey); } else { client->SetRes(CmdRes::kSyntaxErr, "ltrim cmd error"); } @@ -254,6 +274,8 @@ void LSetCmd::DoCmd(PClient* client) { PSTORE.GetBackend(client->GetCurrentDB())->GetStorage()->LSet(client->Key(), val, client->argv_[3]); if (s.ok()) { client->SetRes(CmdRes::kOK); + } else if (s.ToString() == ErrTypeMessage) { + client->SetRes(CmdRes::kmultikey); } else if (s.IsNotFound()) { client->SetRes(CmdRes::kNotFound); } else if (s.IsCorruption()) { @@ -287,6 +309,10 @@ void LInsertCmd::DoCmd(PClient* client) { storage::Status s = PSTORE.GetBackend(client->GetCurrentDB()) ->GetStorage() ->LInsert(client->Key(), before_or_after, client->argv_[3], client->argv_[4], &ret); + if (s.ToString() == ErrTypeMessage) { + client->SetRes(CmdRes::kmultikey); + return; + } if (!s.ok() && s.IsNotFound()) { client->SetRes(CmdRes::kSyntaxErr, "linsert cmd error"); // just a safeguard return; @@ -314,6 +340,8 @@ void LIndexCmd::DoCmd(PClient* client) { storage::Status s = PSTORE.GetBackend(client->GetCurrentDB())->GetStorage()->LIndex(client->Key(), freq_, &value); if (s.ok()) { client->AppendString(value); + } else if (s.ToString() == ErrTypeMessage) { + client->SetRes(CmdRes::kmultikey); } else if (s.IsNotFound()) { client->AppendStringLen(-1); } else { @@ -334,6 +362,8 @@ void LLenCmd::DoCmd(PClient* client) { storage::Status s = PSTORE.GetBackend(client->GetCurrentDB())->GetStorage()->LLen(client->Key(), &llen); if (s.ok() || s.IsNotFound()) { client->AppendInteger(static_cast(llen)); + } else if (s.ToString() == ErrTypeMessage) { + client->SetRes(CmdRes::kmultikey); } else { client->SetRes(CmdRes::kErrOther, s.ToString()); } diff --git a/src/cmd_set.cc b/src/cmd_set.cc index 8a750dbb7..7cb54e094 100644 --- a/src/cmd_set.cc +++ b/src/cmd_set.cc @@ -22,8 +22,12 @@ bool SIsMemberCmd::DoInitial(PClient* client) { } void SIsMemberCmd::DoCmd(PClient* client) { int32_t reply_Num = 0; // only change to 1 if ismember . key not exist it is 0 - PSTORE.GetBackend(client->GetCurrentDB())->GetStorage()->SIsmember(client->Key(), client->argv_[2], &reply_Num); - + storage::Status s = + PSTORE.GetBackend(client->GetCurrentDB())->GetStorage()->SIsmember(client->Key(), client->argv_[2], &reply_Num); + if (s.ToString() == ErrTypeMessage) { + client->SetRes(CmdRes::kmultikey); + return; + } client->AppendInteger(reply_Num); } @@ -42,6 +46,8 @@ void SAddCmd::DoCmd(PClient* client) { storage::Status s = PSTORE.GetBackend(client->GetCurrentDB())->GetStorage()->SAdd(client->Key(), members, &ret); if (s.ok()) { client->AppendInteger(ret); + } else if (s.ToString() == ErrTypeMessage) { + client->SetRes(CmdRes::kmultikey); } else { client->SetRes(CmdRes::kSyntaxErr, "sadd cmd error"); } @@ -63,9 +69,15 @@ void SUnionStoreCmd::DoCmd(PClient* client) { storage::Status s = PSTORE.GetBackend(client->GetCurrentDB()) ->GetStorage() ->SUnionstore(client->Keys().at(0), keys, value_to_dest, &ret); + + if (s.ToString() == ErrTypeMessage) { + client->SetRes(CmdRes::kmultikey); + return; + } if (!s.ok()) { client->SetRes(CmdRes::kSyntaxErr, "sunionstore cmd error"); } + client->AppendInteger(ret); } SInterCmd::SInterCmd(const std::string& name, int16_t arity) @@ -81,6 +93,10 @@ bool SInterCmd::DoInitial(PClient* client) { void SInterCmd::DoCmd(PClient* client) { std::vector res_vt; storage::Status s = PSTORE.GetBackend(client->GetCurrentDB())->GetStorage()->SInter(client->Keys(), &res_vt); + if (s.ToString() == ErrTypeMessage) { + client->SetRes(CmdRes::kmultikey); + return; + } if (!s.ok()) { client->SetRes(CmdRes::kErrOther, "sinter cmd error"); return; @@ -101,6 +117,10 @@ void SRemCmd::DoCmd(PClient* client) { int32_t reply_num = 0; storage::Status s = PSTORE.GetBackend(client->GetCurrentDB())->GetStorage()->SRem(client->Key(), to_delete_members, &reply_num); + if (s.ToString() == ErrTypeMessage) { + client->SetRes(CmdRes::kmultikey); + return; + } if (!s.ok()) { client->SetRes(CmdRes::kErrOther, "srem cmd error"); } @@ -119,6 +139,10 @@ bool SUnionCmd::DoInitial(PClient* client) { void SUnionCmd::DoCmd(PClient* client) { std::vector res_vt; storage::Status s = PSTORE.GetBackend(client->GetCurrentDB())->GetStorage()->SUnion(client->Keys(), &res_vt); + if (s.ToString() == ErrTypeMessage) { + client->SetRes(CmdRes::kmultikey); + return; + } if (!s.ok()) { client->SetRes(CmdRes::kErrOther, "sunion cmd error"); } @@ -141,6 +165,10 @@ void SInterStoreCmd::DoCmd(PClient* client) { storage::Status s = PSTORE.GetBackend(client->GetCurrentDB()) ->GetStorage() ->SInterstore(client->Key(), inter_keys, value_to_dest, &reply_num); + if (s.ToString() == ErrTypeMessage) { + client->SetRes(CmdRes::kmultikey); + return; + } if (!s.ok()) { client->SetRes(CmdRes::kSyntaxErr, "sinterstore cmd error"); return; @@ -158,6 +186,10 @@ bool SCardCmd::DoInitial(PClient* client) { void SCardCmd::DoCmd(PClient* client) { int32_t reply_Num = 0; storage::Status s = PSTORE.GetBackend(client->GetCurrentDB())->GetStorage()->SCard(client->Key(), &reply_Num); + if (s.ToString() == ErrTypeMessage) { + client->SetRes(CmdRes::kmultikey); + return; + } if (!s.ok()) { client->SetRes(CmdRes::kSyntaxErr, "scard cmd error"); return; @@ -175,6 +207,10 @@ void SMoveCmd::DoCmd(PClient* client) { storage::Status s = PSTORE.GetBackend(client->GetCurrentDB()) ->GetStorage() ->SMove(client->argv_[1], client->argv_[2], client->argv_[3], &reply_num); + if (s.ToString() == ErrTypeMessage) { + client->SetRes(CmdRes::kmultikey); + return; + } if (!s.ok()) { client->SetRes(CmdRes::kErrOther, "smove cmd error"); return; @@ -204,6 +240,10 @@ void SRandMemberCmd::DoCmd(PClient* client) { std::vector vec_ret; storage::Status s = PSTORE.GetBackend(client->GetCurrentDB())->GetStorage()->SRandmember(client->Key(), this->num_rand, &vec_ret); + if (s.ToString() == ErrTypeMessage) { + client->SetRes(CmdRes::kmultikey); + return; + } if (!s.ok()) { client->SetRes(CmdRes::kSyntaxErr, "srandmember cmd error"); return; @@ -231,6 +271,10 @@ void SPopCmd::DoCmd(PClient* client) { std::vector delete_member; storage::Status s = PSTORE.GetBackend(client->GetCurrentDB())->GetStorage()->SPop(client->Key(), &delete_member, cnt); + if (s.ToString() == ErrTypeMessage) { + client->SetRes(CmdRes::kmultikey); + return; + } if (!s.ok()) { client->SetRes(CmdRes::kSyntaxErr, "spop cmd error"); return; @@ -269,6 +313,10 @@ bool SMembersCmd::DoInitial(PClient* client) { void SMembersCmd::DoCmd(PClient* client) { std::vector delete_members; storage::Status s = PSTORE.GetBackend(client->GetCurrentDB())->GetStorage()->SMembers(client->Key(), &delete_members); + if (s.ToString() == ErrTypeMessage) { + client->SetRes(CmdRes::kmultikey); + return; + } if (!s.ok()) { client->SetRes(CmdRes::kSyntaxErr, "smembers cmd error"); return; @@ -288,6 +336,10 @@ void SDiffCmd::DoCmd(PClient* client) { std::vector diff_members; std::vector diff_keys(client->argv_.begin() + 1, client->argv_.end()); storage::Status s = PSTORE.GetBackend(client->GetCurrentDB())->GetStorage()->SDiff(diff_keys, &diff_members); + if (s.ToString() == ErrTypeMessage) { + client->SetRes(CmdRes::kmultikey); + return; + } if (!s.ok()) { client->SetRes(CmdRes::kSyntaxErr, "sdiff cmd error"); return; @@ -310,6 +362,10 @@ void SDiffstoreCmd::DoCmd(PClient* client) { storage::Status s = PSTORE.GetBackend(client->GetCurrentDB()) ->GetStorage() ->SDiffstore(client->Key(), diffstore_keys, value_to_dest, &reply_num); + if (s.ToString() == ErrTypeMessage) { + client->SetRes(CmdRes::kmultikey); + return; + } if (!s.ok()) { client->SetRes(CmdRes::kSyntaxErr, "sdiffstore cmd error"); return; @@ -363,6 +419,10 @@ void SScanCmd::DoCmd(PClient* client) { auto status = PSTORE.GetBackend(client->GetCurrentDB()) ->GetStorage() ->SScan(client->Key(), cursor, pattern, count, &members, &next_cursor); + if (status.ToString() == ErrTypeMessage) { + client->SetRes(CmdRes::kmultikey); + return; + } if (!status.ok() && !status.IsNotFound()) { client->SetRes(CmdRes::kErrOther, status.ToString()); return; diff --git a/src/cmd_zset.cc b/src/cmd_zset.cc index 8fc8d02fb..27d013e04 100644 --- a/src/cmd_zset.cc +++ b/src/cmd_zset.cc @@ -112,6 +112,8 @@ void ZAddCmd::DoCmd(PClient* client) { PSTORE.GetBackend(client->GetCurrentDB())->GetStorage()->ZAdd(client->Key(), score_members_, &count); if (s.ok()) { client->AppendInteger(count); + } else if (s.ToString() == ErrTypeMessage) { + client->SetRes(CmdRes::kmultikey); } else { client->SetRes(CmdRes::kErrOther, s.ToString()); } @@ -168,6 +170,8 @@ void ZRevrangeCmd::DoCmd(PClient* client) { client->AppendContent(sm.member); } } + } else if (s.ToString() == ErrTypeMessage) { + client->SetRes(CmdRes::kmultikey); } else { client->SetRes(CmdRes::kErrOther, s.ToString()); } @@ -227,6 +231,11 @@ void ZRangebyscoreCmd::DoCmd(PClient* client) { storage::Status s = PSTORE.GetBackend(client->GetCurrentDB()) ->GetStorage() ->ZRangebyscore(client->Key(), min_score, max_score, left_close, right_close, &score_members); + + if (s.ToString() == ErrTypeMessage) { + client->SetRes(CmdRes::kmultikey); + return; + } if (!s.ok() && !s.IsNotFound()) { client->SetRes(CmdRes::kErrOther, s.ToString()); return; @@ -280,6 +289,8 @@ void ZRemrangebyrankCmd::DoCmd(PClient* client) { s = PSTORE.GetBackend(client->GetCurrentDB())->GetStorage()->ZRemrangebyrank(client->Key(), start, end, &ret); if (s.ok() || s.IsNotFound()) { client->AppendInteger(ret); + } else if (s.ToString() == ErrTypeMessage) { + client->SetRes(CmdRes::kmultikey); } else { client->SetRes(CmdRes::kErrOther, s.ToString()); } @@ -343,6 +354,10 @@ void ZRevrangebyscoreCmd::DoCmd(PClient* client) { ->GetStorage() ->ZRevrangebyscore(client->Key(), min_score, max_score, left_close, right_close, count, offset, &score_members); + if (s.ToString() == ErrTypeMessage) { + client->SetRes(CmdRes::kmultikey); + return; + } if (!s.ok() && !s.IsNotFound()) { client->SetRes(CmdRes::kErrOther, s.ToString()); return; @@ -381,6 +396,10 @@ bool ZCardCmd::DoInitial(PClient* client) { void ZCardCmd::DoCmd(PClient* client) { int32_t reply_Num = 0; storage::Status s = PSTORE.GetBackend(client->GetCurrentDB())->GetStorage()->ZCard(client->Key(), &reply_Num); + if (s.ToString() == ErrTypeMessage) { + client->SetRes(CmdRes::kmultikey); + return; + } if (!s.ok()) { client->SetRes(CmdRes::kSyntaxErr, "ZCard cmd error"); return; @@ -493,6 +512,10 @@ void ZRangeCmd::DoCmd(PClient* client) { ->ZRevrange(client->Key(), start, stop, &score_members); } } + if (s.ToString() == ErrTypeMessage) { + client->SetRes(CmdRes::kmultikey); + return; + } if (!s.ok() && !s.IsNotFound()) { client->SetRes(CmdRes::kErrOther, s.ToString()); return; @@ -546,6 +569,8 @@ void ZScoreCmd::DoCmd(PClient* client) { s = PSTORE.GetBackend(client->GetCurrentDB())->GetStorage()->ZScore(client->Key(), client->argv_[2], &score); if (s.ok() || s.IsNotFound()) { client->AppendString(std::to_string(score)); + } else if (s.ToString() == ErrTypeMessage) { + client->SetRes(CmdRes::kmultikey); } else { client->SetRes(CmdRes::kErrOther, s.ToString()); } @@ -596,6 +621,10 @@ void ZRangebylexCmd::DoCmd(PClient* client) { s = PSTORE.GetBackend(client->GetCurrentDB()) ->GetStorage() ->ZRangebylex(client->Key(), min_member, max_member, left_close, right_close, &members); + if (s.ToString() == ErrTypeMessage) { + client->SetRes(CmdRes::kmultikey); + return; + } if (!s.ok() && !s.IsNotFound()) { client->SetRes(CmdRes::kErrOther, s.ToString()); return; @@ -653,6 +682,10 @@ void ZRevrangebylexCmd::DoCmd(PClient* client) { s = PSTORE.GetBackend(client->GetCurrentDB()) ->GetStorage() ->ZRangebylex(client->Key(), min_member, max_member, left_close, right_close, &members); + if (s.ToString() == ErrTypeMessage) { + client->SetRes(CmdRes::kmultikey); + return; + } if (!s.ok() && !s.IsNotFound()) { client->SetRes(CmdRes::kErrOther, s.ToString()); return; @@ -684,6 +717,8 @@ void ZRankCmd::DoCmd(PClient* client) { client->AppendInteger(rank); } else if (s.IsNotFound()) { client->AppendContent("$-1"); + } else if (s.ToString() == ErrTypeMessage) { + client->SetRes(CmdRes::kmultikey); } else { client->SetRes(CmdRes::kErrOther, s.ToString()); } @@ -705,6 +740,8 @@ void ZRevrankCmd::DoCmd(PClient* client) { client->AppendInteger(revrank); } else if (s.IsNotFound()) { client->AppendContent("$-1"); + } else if (s.ToString() == ErrTypeMessage) { + client->SetRes(CmdRes::kmultikey); } else { client->SetRes(CmdRes::kErrOther, s.ToString()); } @@ -725,6 +762,8 @@ void ZRemCmd::DoCmd(PClient* client) { storage::Status s = PSTORE.GetBackend(client->GetCurrentDB())->GetStorage()->ZRem(client->Key(), members, &deleted); if (s.ok() || s.IsNotFound()) { client->AppendInteger(deleted); + } else if (s.ToString() == ErrTypeMessage) { + client->SetRes(CmdRes::kmultikey); } else { client->SetRes(CmdRes::kErrOther, s.ToString()); } @@ -754,6 +793,8 @@ void ZIncrbyCmd::DoCmd(PClient* client) { int64_t len = pstd::D2string(buf, sizeof(buf), score); client->AppendStringLen(len); client->AppendContent(buf); + } else if (s.ToString() == ErrTypeMessage) { + client->SetRes(CmdRes::kmultikey); } else { client->SetRes(CmdRes::kErrOther, s.ToString()); } @@ -784,6 +825,8 @@ void ZRemrangebyscoreCmd::DoCmd(PClient* client) { ->ZRemrangebyscore(client->Key(), min_score, max_score, left_close, right_close, &s_ret); if (s.ok()) { client->AppendInteger(s_ret); + } else if (s.ToString() == ErrTypeMessage) { + client->SetRes(CmdRes::kmultikey); } else { client->SetRes(CmdRes::kErrOther, s.ToString()); } diff --git a/src/storage/CMakeLists.txt b/src/storage/CMakeLists.txt index 48773729d..d7c164699 100644 --- a/src/storage/CMakeLists.txt +++ b/src/storage/CMakeLists.txt @@ -13,9 +13,16 @@ ADD_LIBRARY(storage ${STORAGE_SRC}) TARGET_INCLUDE_DIRECTORIES(storage PUBLIC ${CMAKE_SOURCE_DIR}/src +<<<<<<< HEAD + PUBLIC ${CMAKE_CURRENT_SOURCE_DIR} + PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/include + PUBLIC ${CMAKE_SOURCE_DIR}/src/pstd + PRIVATE ${rocksdb_SOURCE_DIR} +======= PUBLIC ${CMAKE_CURRENT_SOURCE_DIR} PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/include PRIVATE ${rocksdb_SOURCE_DIR}/ +>>>>>>> 128d408618734b08888b9ef3eec85f33c8a872f5 PRIVATE ${rocksdb_SOURCE_DIR}/include PRIVATE ${PROTO_OUTPUT_DIR} ) diff --git a/src/storage/include/storage/storage.h b/src/storage/include/storage/storage.h index 72ad8ea94..d5a919b8f 100644 --- a/src/storage/include/storage/storage.h +++ b/src/storage/include/storage/storage.h @@ -264,7 +264,6 @@ class Storage { // return 0 if the key doesn't exist And del fail // return -1 if the key currently does not hold the given value And del fail Status Delvx(const Slice& key, const Slice& value, int32_t* ret); - // Set key to hold string value if key does not exist // return the length of the string after it was modified by the command Status Setrange(const Slice& key, int64_t start_offset, const Slice& value, int32_t* ret); @@ -381,6 +380,10 @@ class Storage { // or key does not exist. Status HExists(const Slice& key, const Slice& field); + // Returns if field is an existing field at key. + // Return Status::Ok() if the contains field. + // Return Status::NotFound() if does not contain field, + // or key does not exist. // Increments the number stored at field in the hash stored at key by // increment. If key does not exist, a new key holding a hash is created. If // field does not exist the value is set to 0 before the operation is @@ -1039,18 +1042,18 @@ class Storage { // return -1 operation exception errors happen in database // return 0 if key does not exist or does not have an associated timeout // return >=1 if the timueout was set - int32_t Persist(const Slice& key, std::map* type_status); + int32_t Persist(const Slice& key); // Returns the remaining time to live of a key that has a timeout. // return -3 operation exception errors happen in database // return -2 if the key does not exist // return -1 if the key exists but has not associated expire // return > 0 TTL in seconds - std::map TTL(const Slice& key, std::map* type_status); + int32_t TTL(const Slice& key); // Reutrns the data all type of the key // if single is true, the query will return the first one - Status GetType(const std::string& key, bool single, std::vector& types); + Status GetType(const std::string& key, std::string& types); // Reutrns the data all type of the key Status Type(const std::string& key, std::vector& types); diff --git a/src/storage/include/storage/storage_define.h b/src/storage/include/storage/storage_define.h index 4b27c860c..0fae65f48 100644 --- a/src/storage/include/storage/storage_define.h +++ b/src/storage/include/storage/storage_define.h @@ -12,8 +12,6 @@ #include "rocksdb/slice.h" -#include "src/debug.h" - namespace storage { using Slice = rocksdb::Slice; @@ -28,21 +26,17 @@ const int kVersionLength = 8; const int kScoreLength = 8; const int kSuffixReserveLength = 16; const int kListValueIndexLength = 16; - +const int kTypeLength = 1; const int kTimestampLength = 8; enum ColumnFamilyIndex { - kStringsCF = 0, - kHashesMetaCF = 1, - kHashesDataCF = 2, - kSetsMetaCF = 3, - kSetsDataCF = 4, - kListsMetaCF = 5, - kListsDataCF = 6, - kZsetsMetaCF = 7, - kZsetsDataCF = 8, - kZsetsScoreCF = 9, - kColumnFamilyNum = 10, + kMetaCF = 0, + kHashesDataCF = 1, + kSetsDataCF = 2, + kListsDataCF = 3, + kZsetsDataCF = 4, + kZsetsScoreCF = 5, + kColumnFamilyNum = 6, }; const static char kNeedTransformCharacter = '\u0000'; @@ -94,7 +88,7 @@ inline const char* DecodeUserKey(const char* ptr, int length, std::string* user_ for (int idx = 0; idx < length; idx++) { switch (ptr[idx]) { case '\u0000': { - delim_found = zero_ahead ? true : false; + delim_found = zero_ahead; zero_ahead = true; break; } diff --git a/src/storage/src/base_data_value_format.h b/src/storage/src/base_data_value_format.h index b06c59c50..6aeaef854 100644 --- a/src/storage/src/base_data_value_format.h +++ b/src/storage/src/base_data_value_format.h @@ -25,7 +25,7 @@ namespace storage { */ class BaseDataValue : public InternalValue { public: - explicit BaseDataValue(const rocksdb::Slice& user_value) : InternalValue(user_value) {} + explicit BaseDataValue(const rocksdb::Slice& user_value) : InternalValue(Type::kNulltype, user_value) {} virtual ~BaseDataValue() {} virtual rocksdb::Slice Encode() { diff --git a/src/storage/src/base_filter.h b/src/storage/src/base_filter.h index d2c7a629f..ce29ad641 100644 --- a/src/storage/src/base_filter.h +++ b/src/storage/src/base_filter.h @@ -10,11 +10,14 @@ #include #include +#include "pstd/log.h" #include "rocksdb/compaction_filter.h" #include "rocksdb/db.h" #include "src/base_data_key_format.h" #include "src/base_meta_value_format.h" -#include "src/debug.h" +#include "src/lists_meta_value_format.h" +#include "src/strings_value_format.h" +#include "src/zsets_data_key_format.h" namespace storage { @@ -25,23 +28,54 @@ class BaseMetaFilter : public rocksdb::CompactionFilter { bool* value_changed) const override { int64_t unix_time; rocksdb::Env::Default()->GetCurrentTime(&unix_time); - auto cur_time = static_cast(unix_time); - ParsedBaseMetaValue parsed_base_meta_value(value); - TRACE("==========================START=========================="); - TRACE("[MetaFilter], key: %s, count = %d, timestamp: %llu, cur_time: %d, version: %llu", key.ToString().c_str(), - parsed_base_meta_value.Count(), parsed_base_meta_value.Etime(), cur_time, parsed_base_meta_value.Version()); - - if (parsed_base_meta_value.Etime() != 0 && parsed_base_meta_value.Etime() < cur_time && - parsed_base_meta_value.Version() < cur_time) { - TRACE("Drop[Stale & version < cur_time]"); - return true; - } - if (parsed_base_meta_value.Count() == 0 && parsed_base_meta_value.Version() < cur_time) { - TRACE("Drop[Empty & version < cur_time]"); - return true; + auto cur_time = static_cast(unix_time); + auto type = static_cast(static_cast(value[0])); + DEBUG("==========================START=========================="); + if (type == Type::kString) { + ParsedStringsValue parsed_strings_value(value); + DEBUG("[StringsFilter] key: {}, value = {}, timestamp: {}, cur_time: {}", key.ToString().c_str(), + parsed_strings_value.UserValue().ToString().c_str(), parsed_strings_value.Etime(), cur_time); + if (parsed_strings_value.Etime() != 0 && parsed_strings_value.Etime() < cur_time) { + DEBUG("Drop[Stale]"); + return true; + } else { + DEBUG("Reserve"); + return false; + } + } else if (type == Type::kList) { + ParsedListsMetaValue parsed_lists_meta_value(value); + DEBUG("[ListMetaFilter], key: {}, count = {}, timestamp: {}, cur_time: {}, version: {}", key.ToString().c_str(), + parsed_lists_meta_value.Count(), parsed_lists_meta_value.Etime(), cur_time, + parsed_lists_meta_value.Version()); + + if (parsed_lists_meta_value.Etime() != 0 && parsed_lists_meta_value.Etime() < cur_time && + parsed_lists_meta_value.Version() < cur_time) { + DEBUG("Drop[Stale & version < cur_time]"); + return true; + } + if (parsed_lists_meta_value.Count() == 0 && parsed_lists_meta_value.Version() < cur_time) { + DEBUG("Drop[Empty & version < cur_time]"); + return true; + } + DEBUG("Reserve"); + return false; + } else { + ParsedBaseMetaValue parsed_base_meta_value(value); + DEBUG("[MetaFilter] key: {}, count = {}, timestamp: {}, cur_time: {}, version: {}", key.ToString().c_str(), + parsed_base_meta_value.Count(), parsed_base_meta_value.Etime(), cur_time, parsed_base_meta_value.Version()); + + if (parsed_base_meta_value.Etime() != 0 && parsed_base_meta_value.Etime() < cur_time && + parsed_base_meta_value.Version() < cur_time) { + DEBUG("Drop[Stale & version < cur_time]"); + return true; + } + if (parsed_base_meta_value.Count() == 0 && parsed_base_meta_value.Version() < cur_time) { + DEBUG("Drop[Empty & version < cur_time]"); + return true; + } + DEBUG("Reserve"); + return false; } - TRACE("Reserve"); - return false; } const char* Name() const override { return "BaseMetaFilter"; } @@ -69,8 +103,8 @@ class BaseDataFilter : public rocksdb::CompactionFilter { UNUSED(new_value); UNUSED(value_changed); ParsedBaseDataKey parsed_base_data_key(key); - TRACE("==========================START=========================="); - TRACE("[DataFilter], key: %s, data = %s, version = %llu", parsed_base_data_key.Key().ToString().c_str(), + DEBUG("==========================START=========================="); + DEBUG("[DataFilter], key: {}, data = {}, version = {}", parsed_base_data_key.Key().ToString().c_str(), parsed_base_data_key.Data().ToString().c_str(), parsed_base_data_key.Version()); const char* ptr = key.data(); @@ -96,28 +130,28 @@ class BaseDataFilter : public rocksdb::CompactionFilter { meta_not_found_ = true; } else { cur_key_ = ""; - TRACE("Reserve[Get meta_key faild]"); + DEBUG("Reserve[Get meta_key faild]"); return false; } } if (meta_not_found_) { - TRACE("Drop[Meta key not exist]"); + DEBUG("Drop[Meta key not exist]"); return true; } int64_t unix_time; rocksdb::Env::Default()->GetCurrentTime(&unix_time); if (cur_meta_etime_ != 0 && cur_meta_etime_ < static_cast(unix_time)) { - TRACE("Drop[Timeout]"); + DEBUG("Drop[Timeout]"); return true; } if (cur_meta_version_ > parsed_base_data_key.Version()) { - TRACE("Drop[data_key_version < cur_meta_version]"); + DEBUG("Drop[data_key_version < cur_meta_version]"); return true; } else { - TRACE("Reserve[data_key_version == cur_meta_version]"); + DEBUG("Reserve[data_key_version == cur_meta_version]"); return false; } } @@ -166,5 +200,7 @@ using ZSetsMetaFilterFactory = BaseMetaFilterFactory; using ZSetsDataFilter = BaseDataFilter; using ZSetsDataFilterFactory = BaseDataFilterFactory; +using MetaFilter = BaseMetaFilter; +using MetaFilterFactory = BaseMetaFilterFactory; } // namespace storage #endif // SRC_BASE_FILTER_H_ diff --git a/src/storage/src/base_meta_value_format.h b/src/storage/src/base_meta_value_format.h index 12a5e12a7..4c7104f31 100644 --- a/src/storage/src/base_meta_value_format.h +++ b/src/storage/src/base_meta_value_format.h @@ -14,20 +14,20 @@ namespace storage { /* - * | value | version | reserve | cdate | timestamp | - * | | 8B | 16B | 8B | 8B | + * type | value | version | reserve | cdate | timestamp | + * 1B | | 8B | 16B | 8B | 8B | */ // TODO(wangshaoyi): reformat encode, AppendTimestampAndVersion class BaseMetaValue : public InternalValue { public: - explicit BaseMetaValue(const Slice& user_value) : InternalValue(user_value) {} + explicit BaseMetaValue(Type type, const Slice& user_value) : InternalValue(type, user_value) {} rocksdb::Slice Encode() override { size_t usize = user_value_.size(); - size_t needed = usize + kVersionLength + kSuffixReserveLength + 2 * kTimestampLength; + size_t needed = kTypeLength + usize + kVersionLength + kSuffixReserveLength + 2 * kTimestampLength; char* dst = ReAllocIfNeeded(needed); - char* start_pos = dst; - + memcpy(dst, &type_, sizeof(type_)); + dst += sizeof(type_); memcpy(dst, user_value_.data(), user_value_.size()); dst += user_value_.size(); EncodeFixed64(dst, version_); @@ -37,8 +37,7 @@ class BaseMetaValue : public InternalValue { EncodeFixed64(dst, ctime_); dst += sizeof(ctime_); EncodeFixed64(dst, etime_); - dst += sizeof(etime_); - return rocksdb::Slice(start_, needed); + return {start_, needed}; } uint64_t UpdateVersion() { @@ -58,8 +57,12 @@ class ParsedBaseMetaValue : public ParsedInternalValue { // Use this constructor after rocksdb::DB::Get(); explicit ParsedBaseMetaValue(std::string* internal_value_str) : ParsedInternalValue(internal_value_str) { if (internal_value_str->size() >= kBaseMetaValueSuffixLength) { - int offset = 0; - user_value_ = Slice(internal_value_str->data(), internal_value_str->size() - kBaseMetaValueSuffixLength); + size_t offset = 0; + type_ = static_cast(static_cast((*internal_value_str)[0])); + offset += kTypeLength; + + user_value_ = + Slice(internal_value_str->data() + offset, internal_value_str->size() - kBaseMetaValueSuffixLength - offset); offset += user_value_.size(); version_ = DecodeFixed64(internal_value_str->data() + offset); offset += sizeof(version_); @@ -69,14 +72,17 @@ class ParsedBaseMetaValue : public ParsedInternalValue { offset += sizeof(ctime_); etime_ = DecodeFixed64(internal_value_str->data() + offset); } - count_ = DecodeFixed32(internal_value_str->data()); + count_ = DecodeFixed32(internal_value_str->data() + kTypeLength); } // Use this constructor in rocksdb::CompactionFilter::Filter(); explicit ParsedBaseMetaValue(const Slice& internal_value_slice) : ParsedInternalValue(internal_value_slice) { if (internal_value_slice.size() >= kBaseMetaValueSuffixLength) { - int offset = 0; - user_value_ = Slice(internal_value_slice.data(), internal_value_slice.size() - kBaseMetaValueSuffixLength); + size_t offset = 0; + type_ = static_cast(static_cast(internal_value_slice[0])); + offset += kTypeLength; + user_value_ = Slice(internal_value_slice.data() + offset, + internal_value_slice.size() - kBaseMetaValueSuffixLength - offset); offset += user_value_.size(); version_ = DecodeFixed64(internal_value_slice.data() + offset); offset += sizeof(uint64_t); @@ -86,7 +92,7 @@ class ParsedBaseMetaValue : public ParsedInternalValue { offset += sizeof(ctime_); etime_ = DecodeFixed64(internal_value_slice.data() + offset); } - count_ = DecodeFixed32(internal_value_slice.data()); + count_ = DecodeFixed32(internal_value_slice.data() + kTypeLength); } void StripSuffix() override { @@ -125,12 +131,7 @@ class ParsedBaseMetaValue : public ParsedInternalValue { bool IsValid() override { return !IsStale() && Count() != 0; } - bool check_set_count(size_t count) { - if (count > INT32_MAX) { - return false; - } - return true; - } + bool check_set_count(size_t count) { return count <= INT32_MAX; } int32_t Count() { return count_; } @@ -138,7 +139,7 @@ class ParsedBaseMetaValue : public ParsedInternalValue { count_ = count; if (value_) { char* dst = const_cast(value_->data()); - EncodeFixed32(dst, count_); + EncodeFixed32(dst + kTypeLength, count_); } } @@ -155,7 +156,7 @@ class ParsedBaseMetaValue : public ParsedInternalValue { count_ += delta; if (value_) { char* dst = const_cast(value_->data()); - EncodeFixed32(dst, count_); + EncodeFixed32(dst + kTypeLength, count_); } } diff --git a/src/storage/src/base_value_format.h b/src/storage/src/base_value_format.h index 7654c11e4..318fc38ac 100644 --- a/src/storage/src/base_value_format.h +++ b/src/storage/src/base_value_format.h @@ -20,9 +20,11 @@ namespace storage { using Status = rocksdb::Status; using Slice = rocksdb::Slice; +enum class Type : uint8_t { kString = 0, kHash = 1, kList = 2, kSet = 3, kZset = 4, kNulltype = 5 }; + class InternalValue { public: - explicit InternalValue(const Slice& user_value) : user_value_(user_value) {} + explicit InternalValue(Type type, const Slice& user_value) : type_(type), user_value_(user_value) {} virtual ~InternalValue() { if (start_ != space_) { delete[] start_; @@ -60,6 +62,7 @@ class InternalValue { protected: char space_[200]; char* start_ = nullptr; + Type type_; Slice user_value_; uint64_t version_ = 0; uint64_t etime_ = 0; @@ -125,11 +128,14 @@ class ParsedInternalValue { virtual void StripSuffix() = 0; + bool IsSameType(const Type type) { return type_ == type; } + protected: virtual void SetVersionToValue() = 0; virtual void SetEtimeToValue() = 0; virtual void SetCtimeToValue() = 0; std::string* value_ = nullptr; + Type type_; Slice user_value_; uint64_t version_ = 0; uint64_t ctime_ = 0; diff --git a/src/storage/src/custom_comparator.h b/src/storage/src/custom_comparator.h index 7a11c97d0..aea196dda 100644 --- a/src/storage/src/custom_comparator.h +++ b/src/storage/src/custom_comparator.h @@ -9,7 +9,6 @@ #include "rocksdb/comparator.h" #include "src/coding.h" -#include "src/debug.h" #include "storage/storage_define.h" namespace storage { @@ -67,8 +66,6 @@ class ListsDataKeyComparatorImpl : public rocksdb::Comparator { uint64_t index_a = DecodeFixed64(ptr_a); uint64_t index_b = DecodeFixed64(ptr_b); - ptr_a += sizeof(uint64_t); - ptr_b += sizeof(uint64_t); if (index_a != index_b) { return index_a < index_b ? -1 : 1; } else { @@ -84,8 +81,8 @@ class ListsDataKeyComparatorImpl : public rocksdb::Comparator { }; /* zset score key pattern - * | | | | | | | - * | 8 Bytes | Key Size Bytes | 8 Bytes | 8 Bytes | | 16B | + * | Reserve 1 | Key | Version | Score> | Member | Reserve2 | + * | 8 Bytes |Key Size Bytes| 8 Bytes | 8 Bytes | | 16B | */ class ZSetsScoreKeyComparatorImpl : public rocksdb::Comparator { public: diff --git a/src/storage/src/lists_filter.h b/src/storage/src/lists_filter.h index 811b71632..e40867120 100644 --- a/src/storage/src/lists_filter.h +++ b/src/storage/src/lists_filter.h @@ -12,52 +12,10 @@ #include "rocksdb/compaction_filter.h" #include "rocksdb/db.h" -#include "src/debug.h" #include "src/lists_data_key_format.h" #include "src/lists_meta_value_format.h" namespace storage { - -class ListsMetaFilter : public rocksdb::CompactionFilter { - public: - ListsMetaFilter() = default; - bool Filter(int level, const rocksdb::Slice& key, const rocksdb::Slice& value, std::string* new_value, - bool* value_changed) const override { - int64_t unix_time; - rocksdb::Env::Default()->GetCurrentTime(&unix_time); - auto cur_time = static_cast(unix_time); - ParsedListsMetaValue parsed_lists_meta_value(value); - TRACE("==========================START=========================="); - TRACE("[ListMetaFilter], key: %s, count = %llu, timestamp: %llu, cur_time: %d, version: %llu", - key.ToString().c_str(), parsed_lists_meta_value.Count(), parsed_lists_meta_value.Etime(), cur_time, - parsed_lists_meta_value.Version()); - - if (parsed_lists_meta_value.Etime() != 0 && parsed_lists_meta_value.Etime() < cur_time && - parsed_lists_meta_value.Version() < cur_time) { - TRACE("Drop[Stale & version < cur_time]"); - return true; - } - if (parsed_lists_meta_value.Count() == 0 && parsed_lists_meta_value.Version() < cur_time) { - TRACE("Drop[Empty & version < cur_time]"); - return true; - } - TRACE("Reserve"); - return false; - } - - const char* Name() const override { return "ListsMetaFilter"; } -}; - -class ListsMetaFilterFactory : public rocksdb::CompactionFilterFactory { - public: - ListsMetaFilterFactory() = default; - std::unique_ptr CreateCompactionFilter( - const rocksdb::CompactionFilter::Context& context) override { - return std::unique_ptr(new ListsMetaFilter()); - } - const char* Name() const override { return "ListsMetaFilterFactory"; } -}; - class ListsDataFilter : public rocksdb::CompactionFilter { public: ListsDataFilter(rocksdb::DB* db, std::vector* cf_handles_ptr, int meta_cf_index) @@ -70,8 +28,8 @@ class ListsDataFilter : public rocksdb::CompactionFilter { UNUSED(new_value); UNUSED(value_changed); ParsedListsDataKey parsed_lists_data_key(key); - TRACE("==========================START=========================="); - TRACE("[DataFilter], key: %s, index = %llu, data = %s, version = %llu", + DEBUG("==========================START=========================="); + DEBUG("[DataFilter], key: %s, index = %llu, data = %s, version = %llu", parsed_lists_data_key.key().ToString().c_str(), parsed_lists_data_key.index(), value.ToString().c_str(), parsed_lists_data_key.Version()); @@ -98,28 +56,28 @@ class ListsDataFilter : public rocksdb::CompactionFilter { meta_not_found_ = true; } else { cur_key_ = ""; - TRACE("Reserve[Get meta_key faild]"); + DEBUG("Reserve[Get meta_key faild]"); return false; } } if (meta_not_found_) { - TRACE("Drop[Meta key not exist]"); + DEBUG("Drop[Meta key not exist]"); return true; } int64_t unix_time; rocksdb::Env::Default()->GetCurrentTime(&unix_time); if (cur_meta_etime_ != 0 && cur_meta_etime_ < static_cast(unix_time)) { - TRACE("Drop[Timeout]"); + DEBUG("Drop[Timeout]"); return true; } if (cur_meta_version_ > parsed_lists_data_key.Version()) { - TRACE("Drop[list_data_key_version < cur_meta_version]"); + DEBUG("Drop[list_data_key_version < cur_meta_version]"); return true; } else { - TRACE("Reserve[list_data_key_version == cur_meta_version]"); + DEBUG("Reserve[list_data_key_version == cur_meta_version]"); return false; } } diff --git a/src/storage/src/lists_meta_value_format.h b/src/storage/src/lists_meta_value_format.h index 4117b4bcc..eafdc1759 100644 --- a/src/storage/src/lists_meta_value_format.h +++ b/src/storage/src/lists_meta_value_format.h @@ -17,20 +17,21 @@ const uint64_t InitalLeftIndex = 9223372036854775807; const uint64_t InitalRightIndex = 9223372036854775808U; /* - *| list_size | version | left index | right index | reserve | cdate | timestamp | - *| 8B | 8B | 8B | 8B | 16B | 8B | 8B | + *| type | list_size | version | left index | right index | reserve | cdate | timestamp | + *| 1B | 8B | 8B | 8B | 8B | 16B | 8B | 8B | */ class ListsMetaValue : public InternalValue { public: explicit ListsMetaValue(const rocksdb::Slice& user_value) - : InternalValue(user_value), left_index_(InitalLeftIndex), right_index_(InitalRightIndex) {} + : InternalValue(Type::kList, user_value), left_index_(InitalLeftIndex), right_index_(InitalRightIndex) {} rocksdb::Slice Encode() override { size_t usize = user_value_.size(); - size_t needed = usize + kVersionLength + 2 * kListValueIndexLength + kSuffixReserveLength + 2 * kTimestampLength; + size_t needed = + kTypeLength + usize + kVersionLength + 2 * kListValueIndexLength + kSuffixReserveLength + 2 * kTimestampLength; char* dst = ReAllocIfNeeded(needed); - char* start_pos = dst; - + memcpy(dst, &type_, sizeof(type_)); + dst += sizeof(type_); memcpy(dst, user_value_.data(), usize); dst += usize; EncodeFixed64(dst, version_); @@ -44,7 +45,7 @@ class ListsMetaValue : public InternalValue { EncodeFixed64(dst, ctime_); dst += kTimestampLength; EncodeFixed64(dst, etime_); - return rocksdb::Slice(start_pos, needed); + return {start_, needed}; } uint64_t UpdateVersion() { @@ -77,9 +78,11 @@ class ParsedListsMetaValue : public ParsedInternalValue { explicit ParsedListsMetaValue(std::string* internal_value_str) : ParsedInternalValue(internal_value_str) { assert(internal_value_str->size() >= kListsMetaValueSuffixLength); if (internal_value_str->size() >= kListsMetaValueSuffixLength) { - int offset = 0; - user_value_ = - rocksdb::Slice(internal_value_str->data(), internal_value_str->size() - kListsMetaValueSuffixLength); + size_t offset = 0; + type_ = static_cast(static_cast((*internal_value_str)[0])); + offset += kTypeLength; + user_value_ = rocksdb::Slice(internal_value_str->data() + kTypeLength, + internal_value_str->size() - kListsMetaValueSuffixLength - kTypeLength); offset += user_value_.size(); version_ = DecodeFixed64(internal_value_str->data() + offset); offset += kVersionLength; @@ -94,7 +97,7 @@ class ParsedListsMetaValue : public ParsedInternalValue { etime_ = DecodeFixed64(internal_value_str->data() + offset); offset += kTimestampLength; } - count_ = DecodeFixed64(internal_value_str->data()); + count_ = DecodeFixed64(internal_value_str->data() + kTypeLength); } // Use this constructor in rocksdb::CompactionFilter::Filter(); @@ -102,9 +105,11 @@ class ParsedListsMetaValue : public ParsedInternalValue { : ParsedInternalValue(internal_value_slice) { assert(internal_value_slice.size() >= kListsMetaValueSuffixLength); if (internal_value_slice.size() >= kListsMetaValueSuffixLength) { - int offset = 0; - user_value_ = - rocksdb::Slice(internal_value_slice.data(), internal_value_slice.size() - kListsMetaValueSuffixLength); + size_t offset = 0; + type_ = static_cast(static_cast(internal_value_slice[0])); + offset += kTypeLength; + user_value_ = rocksdb::Slice(internal_value_slice.data() + kTypeLength, + internal_value_slice.size() - kListsMetaValueSuffixLength - kTypeLength); offset += user_value_.size(); version_ = DecodeFixed64(internal_value_slice.data() + offset); offset += kVersionLength; @@ -117,9 +122,8 @@ class ParsedListsMetaValue : public ParsedInternalValue { ctime_ = DecodeFixed64(internal_value_slice.data() + offset); offset += kTimestampLength; etime_ = DecodeFixed64(internal_value_slice.data() + offset); - offset += kTimestampLength; } - count_ = DecodeFixed64(internal_value_slice.data()); + count_ = DecodeFixed64(internal_value_slice.data() + kTypeLength); } void StripSuffix() override { @@ -175,7 +179,7 @@ class ParsedListsMetaValue : public ParsedInternalValue { count_ = count; if (value_) { char* dst = const_cast(value_->data()); - EncodeFixed64(dst, count_); + EncodeFixed64(dst + kTypeLength, count_); } } @@ -183,7 +187,7 @@ class ParsedListsMetaValue : public ParsedInternalValue { count_ += delta; if (value_) { char* dst = const_cast(value_->data()); - EncodeFixed64(dst, count_); + EncodeFixed64(dst + kTypeLength, count_); } } diff --git a/src/storage/src/redis.cc b/src/storage/src/redis.cc index 6417f07bb..895b7eba2 100644 --- a/src/storage/src/redis.cc +++ b/src/storage/src/redis.cc @@ -10,7 +10,6 @@ #include "src/base_filter.h" #include "src/lists_filter.h" #include "src/redis.h" -#include "src/strings_filter.h" #include "src/zsets_filter.h" #define ADD_TABLE_PROPERTY_COLLECTOR_FACTORY(type) \ @@ -71,91 +70,65 @@ Status Redis::Open(const StorageOptions& storage_options, const std::string& db_ // Set up separate configuration for RocksDB rocksdb::DBOptions db_ops(storage_options.options); - // string column-family options - rocksdb::ColumnFamilyOptions string_cf_ops(storage_options.options); - string_cf_ops.compaction_filter_factory = std::make_shared(); - rocksdb::BlockBasedTableOptions string_table_ops(table_ops); + // meta & string column-family options + rocksdb::ColumnFamilyOptions meta_cf_ops(storage_options.options); + // TODO change compaction filter + meta_cf_ops.compaction_filter_factory = std::make_shared(); + rocksdb::BlockBasedTableOptions meta_table_ops(table_ops); if (!storage_options.share_block_cache && (storage_options.block_cache_size > 0)) { - string_table_ops.block_cache = rocksdb::NewLRUCache(storage_options.block_cache_size); + meta_table_ops.block_cache = rocksdb::NewLRUCache(storage_options.block_cache_size); } - string_cf_ops.table_factory.reset(rocksdb::NewBlockBasedTableFactory(string_table_ops)); + meta_cf_ops.table_factory.reset(rocksdb::NewBlockBasedTableFactory(meta_table_ops)); // hash column-family options - rocksdb::ColumnFamilyOptions hash_meta_cf_ops(storage_options.options); rocksdb::ColumnFamilyOptions hash_data_cf_ops(storage_options.options); - hash_meta_cf_ops.compaction_filter_factory = std::make_shared(); - hash_data_cf_ops.compaction_filter_factory = - std::make_shared(&db_, &handles_, kHashesMetaCF); - rocksdb::BlockBasedTableOptions hash_meta_cf_table_ops(table_ops); + hash_data_cf_ops.compaction_filter_factory = std::make_shared(&db_, &handles_, kMetaCF); rocksdb::BlockBasedTableOptions hash_data_cf_table_ops(table_ops); if (!storage_options.share_block_cache && (storage_options.block_cache_size > 0)) { - hash_meta_cf_table_ops.block_cache = rocksdb::NewLRUCache(storage_options.block_cache_size); hash_data_cf_table_ops.block_cache = rocksdb::NewLRUCache(storage_options.block_cache_size); } - hash_meta_cf_ops.table_factory.reset(rocksdb::NewBlockBasedTableFactory(hash_meta_cf_table_ops)); hash_data_cf_ops.table_factory.reset(rocksdb::NewBlockBasedTableFactory(hash_data_cf_table_ops)); // list column-family options - rocksdb::ColumnFamilyOptions list_meta_cf_ops(storage_options.options); rocksdb::ColumnFamilyOptions list_data_cf_ops(storage_options.options); - list_meta_cf_ops.compaction_filter_factory = std::make_shared(); - list_data_cf_ops.compaction_filter_factory = std::make_shared(&db_, &handles_, kListsMetaCF); + list_data_cf_ops.compaction_filter_factory = std::make_shared(&db_, &handles_, kMetaCF); list_data_cf_ops.comparator = ListsDataKeyComparator(); - rocksdb::BlockBasedTableOptions list_meta_cf_table_ops(table_ops); rocksdb::BlockBasedTableOptions list_data_cf_table_ops(table_ops); if (!storage_options.share_block_cache && (storage_options.block_cache_size > 0)) { - list_meta_cf_table_ops.block_cache = rocksdb::NewLRUCache(storage_options.block_cache_size); list_data_cf_table_ops.block_cache = rocksdb::NewLRUCache(storage_options.block_cache_size); } - list_meta_cf_ops.table_factory.reset(rocksdb::NewBlockBasedTableFactory(list_meta_cf_table_ops)); list_data_cf_ops.table_factory.reset(rocksdb::NewBlockBasedTableFactory(list_data_cf_table_ops)); // set column-family options - rocksdb::ColumnFamilyOptions set_meta_cf_ops(storage_options.options); rocksdb::ColumnFamilyOptions set_data_cf_ops(storage_options.options); - set_meta_cf_ops.compaction_filter_factory = std::make_shared(); - set_data_cf_ops.compaction_filter_factory = std::make_shared(&db_, &handles_, kSetsMetaCF); - rocksdb::BlockBasedTableOptions set_meta_cf_table_ops(table_ops); + set_data_cf_ops.compaction_filter_factory = std::make_shared(&db_, &handles_, kMetaCF); rocksdb::BlockBasedTableOptions set_data_cf_table_ops(table_ops); if (!storage_options.share_block_cache && (storage_options.block_cache_size > 0)) { - set_meta_cf_table_ops.block_cache = rocksdb::NewLRUCache(storage_options.block_cache_size); set_data_cf_table_ops.block_cache = rocksdb::NewLRUCache(storage_options.block_cache_size); } - set_meta_cf_ops.table_factory.reset(rocksdb::NewBlockBasedTableFactory(set_meta_cf_table_ops)); set_data_cf_ops.table_factory.reset(rocksdb::NewBlockBasedTableFactory(set_data_cf_table_ops)); // zset column-family options - rocksdb::ColumnFamilyOptions zset_meta_cf_ops(storage_options.options); rocksdb::ColumnFamilyOptions zset_data_cf_ops(storage_options.options); rocksdb::ColumnFamilyOptions zset_score_cf_ops(storage_options.options); - zset_meta_cf_ops.compaction_filter_factory = std::make_shared(); - zset_data_cf_ops.compaction_filter_factory = std::make_shared(&db_, &handles_, kZsetsMetaCF); - zset_score_cf_ops.compaction_filter_factory = - std::make_shared(&db_, &handles_, kZsetsMetaCF); + zset_data_cf_ops.compaction_filter_factory = std::make_shared(&db_, &handles_, kMetaCF); + zset_score_cf_ops.compaction_filter_factory = std::make_shared(&db_, &handles_, kMetaCF); zset_score_cf_ops.comparator = ZSetsScoreKeyComparator(); - rocksdb::BlockBasedTableOptions zset_meta_cf_table_ops(table_ops); rocksdb::BlockBasedTableOptions zset_data_cf_table_ops(table_ops); rocksdb::BlockBasedTableOptions zset_score_cf_table_ops(table_ops); if (!storage_options.share_block_cache && (storage_options.block_cache_size > 0)) { - zset_meta_cf_table_ops.block_cache = rocksdb::NewLRUCache(storage_options.block_cache_size); zset_data_cf_table_ops.block_cache = rocksdb::NewLRUCache(storage_options.block_cache_size); - zset_meta_cf_table_ops.block_cache = rocksdb::NewLRUCache(storage_options.block_cache_size); } - zset_meta_cf_ops.table_factory.reset(rocksdb::NewBlockBasedTableFactory(zset_meta_cf_table_ops)); zset_data_cf_ops.table_factory.reset(rocksdb::NewBlockBasedTableFactory(zset_data_cf_table_ops)); zset_score_cf_ops.table_factory.reset(rocksdb::NewBlockBasedTableFactory(zset_score_cf_table_ops)); if (append_log_function_) { // Add log index table property collector factory to each column family - ADD_TABLE_PROPERTY_COLLECTOR_FACTORY(string); - ADD_TABLE_PROPERTY_COLLECTOR_FACTORY(hash_meta); + ADD_TABLE_PROPERTY_COLLECTOR_FACTORY(meta); ADD_TABLE_PROPERTY_COLLECTOR_FACTORY(hash_data); - ADD_TABLE_PROPERTY_COLLECTOR_FACTORY(list_meta); ADD_TABLE_PROPERTY_COLLECTOR_FACTORY(list_data); - ADD_TABLE_PROPERTY_COLLECTOR_FACTORY(set_meta); ADD_TABLE_PROPERTY_COLLECTOR_FACTORY(set_data); - ADD_TABLE_PROPERTY_COLLECTOR_FACTORY(zset_meta); ADD_TABLE_PROPERTY_COLLECTOR_FACTORY(zset_data); ADD_TABLE_PROPERTY_COLLECTOR_FACTORY(zset_score); @@ -165,18 +138,15 @@ Status Redis::Open(const StorageOptions& storage_options, const std::string& db_ } std::vector column_families; - column_families.emplace_back(rocksdb::kDefaultColumnFamilyName, string_cf_ops); + // meta & string cf + column_families.emplace_back(rocksdb::kDefaultColumnFamilyName, meta_cf_ops); // hash CF - column_families.emplace_back("hash_meta_cf", hash_meta_cf_ops); column_families.emplace_back("hash_data_cf", hash_data_cf_ops); // set CF - column_families.emplace_back("set_meta_cf", set_meta_cf_ops); column_families.emplace_back("set_data_cf", set_data_cf_ops); // list CF - column_families.emplace_back("list_meta_cf", list_meta_cf_ops); column_families.emplace_back("list_data_cf", list_data_cf_ops); // zset CF - column_families.emplace_back("zset_meta_cf", zset_meta_cf_ops); column_families.emplace_back("zset_data_cf", zset_data_cf_ops); column_families.emplace_back("zset_score_cf", zset_score_cf_ops); @@ -228,7 +198,7 @@ Status Redis::CompactRange(const DataType& dtype, const rocksdb::Slice* begin, c break; case DataType::kHashes: if (type == kMeta || type == kMetaAndData) { - s = db_->CompactRange(default_compact_range_options_, handles_[kHashesMetaCF], begin, end); + s = db_->CompactRange(default_compact_range_options_, handles_[kMetaCF], begin, end); } if (s.ok() && (type == kData || type == kMetaAndData)) { s = db_->CompactRange(default_compact_range_options_, handles_[kHashesDataCF], begin, end); @@ -236,7 +206,7 @@ Status Redis::CompactRange(const DataType& dtype, const rocksdb::Slice* begin, c break; case DataType::kSets: if (type == kMeta || type == kMetaAndData) { - db_->CompactRange(default_compact_range_options_, handles_[kSetsMetaCF], begin, end); + db_->CompactRange(default_compact_range_options_, handles_[kMetaCF], begin, end); } if (s.ok() && (type == kData || type == kMetaAndData)) { db_->CompactRange(default_compact_range_options_, handles_[kSetsDataCF], begin, end); @@ -244,7 +214,7 @@ Status Redis::CompactRange(const DataType& dtype, const rocksdb::Slice* begin, c break; case DataType::kLists: if (type == kMeta || type == kMetaAndData) { - s = db_->CompactRange(default_compact_range_options_, handles_[kListsMetaCF], begin, end); + s = db_->CompactRange(default_compact_range_options_, handles_[kMetaCF], begin, end); } if (s.ok() && (type == kData || type == kMetaAndData)) { s = db_->CompactRange(default_compact_range_options_, handles_[kListsDataCF], begin, end); @@ -252,7 +222,7 @@ Status Redis::CompactRange(const DataType& dtype, const rocksdb::Slice* begin, c break; case DataType::kZSets: if (type == kMeta || type == kMetaAndData) { - db_->CompactRange(default_compact_range_options_, handles_[kZsetsMetaCF], begin, end); + db_->CompactRange(default_compact_range_options_, handles_[kMetaCF], begin, end); } if (s.ok() && (type == kData || type == kMetaAndData)) { db_->CompactRange(default_compact_range_options_, handles_[kZsetsDataCF], begin, end); diff --git a/src/storage/src/redis.h b/src/storage/src/redis.h index 24d16dd18..b3317caca 100644 --- a/src/storage/src/redis.h +++ b/src/storage/src/redis.h @@ -18,7 +18,6 @@ #include "pstd/env.h" #include "pstd/log.h" #include "src/custom_comparator.h" -#include "src/debug.h" #include "src/lock_mgr.h" #include "src/lru_cache.h" #include "src/mutex_impl.h" @@ -148,13 +147,17 @@ class Redis { virtual Status ListsPersist(const Slice& key); virtual Status ZsetsPersist(const Slice& key); virtual Status SetsPersist(const Slice& key); + virtual Status Persist(const Slice& key); virtual Status StringsTTL(const Slice& key, uint64_t* timestamp); virtual Status HashesTTL(const Slice& key, uint64_t* timestamp); virtual Status ListsTTL(const Slice& key, uint64_t* timestamp); virtual Status ZsetsTTL(const Slice& key, uint64_t* timestamp); virtual Status SetsTTL(const Slice& key, uint64_t* timestamp); + virtual Status TTL(const Slice& key, uint64_t* timestamp); + virtual Status GetType(const Slice& key, std::string& types); + virtual Status IsExist(const Slice& key); virtual Status StringsRename(const Slice& key, Redis* new_inst, const Slice& newkey); virtual Status HashesRename(const Slice& key, Redis* new_inst, const Slice& newkey); virtual Status ListsRename(const Slice& key, Redis* new_inst, const Slice& newkey); @@ -198,7 +201,10 @@ class Redis { Status BitPos(const Slice& key, int32_t bit, int64_t start_offset, int64_t* ret); Status BitPos(const Slice& key, int32_t bit, int64_t start_offset, int64_t end_offset, int64_t* ret); Status PKSetexAt(const Slice& key, const Slice& value, uint64_t timestamp); - + Status Exists(const Slice& key); + Status Del(const Slice& key); + Status Expire(const Slice& key, uint64_t timestamp); + Status Expireat(const Slice& key, uint64_t timestamp); // Hash Commands Status HDel(const Slice& key, const std::vector& fields, int32_t* ret); Status HExists(const Slice& key, const Slice& field); @@ -335,20 +341,22 @@ class Redis { options.iterate_upper_bound = upper_bound; switch (type) { case 'k': - return new StringsIterator(options, db_, handles_[kStringsCF], pattern); + return new StringsIterator(options, db_, handles_[kMetaCF], pattern); break; case 'h': - return new HashesIterator(options, db_, handles_[kHashesMetaCF], pattern); + return new HashesIterator(options, db_, handles_[kMetaCF], pattern); break; case 's': - return new SetsIterator(options, db_, handles_[kSetsMetaCF], pattern); + return new SetsIterator(options, db_, handles_[kMetaCF], pattern); break; case 'l': - return new ListsIterator(options, db_, handles_[kListsMetaCF], pattern); + return new ListsIterator(options, db_, handles_[kMetaCF], pattern); break; case 'z': - return new ZsetsIterator(options, db_, handles_[kZsetsMetaCF], pattern); + return new ZsetsIterator(options, db_, handles_[kMetaCF], pattern); break; + case 'a': + return new AllIterator(options, db_, handles_[kMetaCF], pattern); default: WARN("Invalid datatype to create iterator"); return nullptr; diff --git a/src/storage/src/redis_hashes.cc b/src/storage/src/redis_hashes.cc index 039a0b356..6a2edb73b 100644 --- a/src/storage/src/redis_hashes.cc +++ b/src/storage/src/redis_hashes.cc @@ -38,9 +38,12 @@ Status Redis::ScanHashesKeyNum(KeyInfo* key_info) { int64_t curtime; rocksdb::Env::Default()->GetCurrentTime(&curtime); - rocksdb::Iterator* iter = db_->NewIterator(iterator_options, handles_[kHashesMetaCF]); + rocksdb::Iterator* iter = db_->NewIterator(iterator_options, handles_[kMetaCF]); for (iter->SeekToFirst(); iter->Valid(); iter->Next()) { ParsedHashesMetaValue parsed_hashes_meta_value(iter->value()); + if (!parsed_hashes_meta_value.IsSameType(Type::kHash)) { + continue; + } if (parsed_hashes_meta_value.IsStale() || parsed_hashes_meta_value.Count() == 0) { invaild_keys++; } else { @@ -72,16 +75,19 @@ Status Redis::HashesPKPatternMatchDel(const std::string& pattern, int32_t* ret) int32_t total_delete = 0; Status s; rocksdb::WriteBatch batch; - rocksdb::Iterator* iter = db_->NewIterator(iterator_options, handles_[kHashesMetaCF]); + rocksdb::Iterator* iter = db_->NewIterator(iterator_options, handles_[kMetaCF]); iter->SeekToFirst(); while (iter->Valid()) { key = iter->key().ToString(); meta_value = iter->value().ToString(); ParsedHashesMetaValue parsed_hashes_meta_value(&meta_value); + if (!parsed_hashes_meta_value.IsSameType(Type::kHash)) { + continue; + } if (!parsed_hashes_meta_value.IsStale() && (parsed_hashes_meta_value.Count() != 0) && (StringMatch(pattern.data(), pattern.size(), key.data(), key.size(), 0) != 0)) { parsed_hashes_meta_value.InitialMetaValue(); - batch.Put(handles_[kHashesMetaCF], key, meta_value); + batch.Put(handles_[kMetaCF], key, meta_value); } if (static_cast(batch.Count()) >= BATCH_DELETE_LIMIT) { s = db_->Write(default_write_options_, &batch); @@ -131,9 +137,13 @@ Status Redis::HDel(const Slice& key, const std::vector& fields, int read_options.snapshot = snapshot; BaseMetaKey base_meta_key(key); - Status s = db_->Get(read_options, handles_[kHashesMetaCF], base_meta_key.Encode(), &meta_value); + Status s = db_->Get(read_options, handles_[kMetaCF], base_meta_key.Encode(), &meta_value); if (s.ok()) { ParsedHashesMetaValue parsed_hashes_meta_value(&meta_value); + auto type = static_cast(static_cast(meta_value[0])); + if (type != Type::kHash) { + return Status::InvalidArgument("WRONGTYPE Operation against a key holding the wrong kind of value"); + } if (parsed_hashes_meta_value.IsStale() || parsed_hashes_meta_value.Count() == 0) { *ret = 0; return Status::OK(); @@ -158,7 +168,7 @@ Status Redis::HDel(const Slice& key, const std::vector& fields, int return Status::InvalidArgument("hash size overflow"); } parsed_hashes_meta_value.ModifyCount(-del_cnt); - batch->Put(kHashesMetaCF, base_meta_key.Encode(), meta_value); + batch->Put(kMetaCF, base_meta_key.Encode(), meta_value); } } else if (s.IsNotFound()) { *ret = 0; @@ -185,9 +195,14 @@ Status Redis::HGet(const Slice& key, const Slice& field, std::string* value) { read_options.snapshot = snapshot; BaseMetaKey base_meta_key(key); - Status s = db_->Get(read_options, handles_[kHashesMetaCF], base_meta_key.Encode(), &meta_value); + Status s = db_->Get(read_options, handles_[kMetaCF], base_meta_key.Encode(), &meta_value); if (s.ok()) { + auto type = static_cast(static_cast(meta_value[0])); + if (type != Type::kHash) { + return Status::InvalidArgument("WRONGTYPE Operation against a key holding the wrong kind of value"); + } ParsedHashesMetaValue parsed_hashes_meta_value(&meta_value); + if (parsed_hashes_meta_value.IsStale()) { return Status::NotFound("Stale"); } else if (parsed_hashes_meta_value.Count() == 0) { @@ -215,8 +230,12 @@ Status Redis::HGetall(const Slice& key, std::vector* fvs) { read_options.snapshot = snapshot; BaseMetaKey base_meta_key(key); - Status s = db_->Get(read_options, handles_[kHashesMetaCF], base_meta_key.Encode(), &meta_value); + Status s = db_->Get(read_options, handles_[kMetaCF], base_meta_key.Encode(), &meta_value); if (s.ok()) { + auto type = static_cast(static_cast(meta_value[0])); + if (type != Type::kHash) { + return Status::InvalidArgument("WRONGTYPE Operation against a key holding the wrong kind of value"); + } ParsedHashesMetaValue parsed_hashes_meta_value(&meta_value); if (parsed_hashes_meta_value.IsStale()) { return Status::NotFound("Stale"); @@ -248,9 +267,14 @@ Status Redis::HGetallWithTTL(const Slice& key, std::vector* fvs, uin ScopeSnapshot ss(db_, &snapshot); read_options.snapshot = snapshot; BaseMetaKey base_meta_key(key); - Status s = db_->Get(read_options, handles_[kHashesMetaCF], base_meta_key.Encode(), &meta_value); + Status s = db_->Get(read_options, handles_[kMetaCF], base_meta_key.Encode(), &meta_value); if (s.ok()) { + auto type = static_cast(static_cast(meta_value[0])); + if (type != Type::kHash) { + return Status::InvalidArgument("WRONGTYPE Operation against a key holding the wrong kind of value"); + } ParsedHashesMetaValue parsed_hashes_meta_value(&meta_value); + if (parsed_hashes_meta_value.Count() == 0) { return Status::NotFound(); } else if (parsed_hashes_meta_value.IsStale()) { @@ -293,16 +317,21 @@ Status Redis::HIncrby(const Slice& key, const Slice& field, int64_t value, int64 std::string meta_value; BaseMetaKey base_meta_key(key); - Status s = db_->Get(default_read_options_, handles_[kHashesMetaCF], base_meta_key.Encode(), &meta_value); + Status s = db_->Get(default_read_options_, handles_[kMetaCF], base_meta_key.Encode(), &meta_value); char value_buf[32] = {0}; char meta_value_buf[4] = {0}; if (s.ok()) { + auto type = static_cast(static_cast(meta_value[0])); + if (type != Type::kHash) { + return Status::InvalidArgument("WRONGTYPE Operation against a key holding the wrong kind of value"); + } ParsedHashesMetaValue parsed_hashes_meta_value(&meta_value); + if (parsed_hashes_meta_value.IsStale() || parsed_hashes_meta_value.Count() == 0) { version = parsed_hashes_meta_value.UpdateVersion(); parsed_hashes_meta_value.SetCount(1); parsed_hashes_meta_value.SetEtime(0); - batch.Put(handles_[kHashesMetaCF], base_meta_key.Encode(), meta_value); + batch.Put(handles_[kMetaCF], base_meta_key.Encode(), meta_value); HashesDataKey hashes_data_key(key, version, field); Int64ToStr(value_buf, 32, value); batch.Put(handles_[kHashesDataCF], hashes_data_key.Encode(), value_buf); @@ -333,7 +362,7 @@ Status Redis::HIncrby(const Slice& key, const Slice& field, int64_t value, int64 } BaseDataValue internal_value(value_buf); parsed_hashes_meta_value.ModifyCount(1); - batch.Put(handles_[kHashesMetaCF], base_meta_key.Encode(), meta_value); + batch.Put(handles_[kMetaCF], base_meta_key.Encode(), meta_value); batch.Put(handles_[kHashesDataCF], hashes_data_key.Encode(), internal_value.Encode()); *ret = value; } else { @@ -342,9 +371,9 @@ Status Redis::HIncrby(const Slice& key, const Slice& field, int64_t value, int64 } } else if (s.IsNotFound()) { EncodeFixed32(meta_value_buf, 1); - HashesMetaValue hashes_meta_value(Slice(meta_value_buf, sizeof(int32_t))); + HashesMetaValue hashes_meta_value(Type::kHash, Slice(meta_value_buf, 4)); version = hashes_meta_value.UpdateVersion(); - batch.Put(handles_[kHashesMetaCF], base_meta_key.Encode(), hashes_meta_value.Encode()); + batch.Put(handles_[kMetaCF], base_meta_key.Encode(), hashes_meta_value.Encode()); HashesDataKey hashes_data_key(key, version, field); Int64ToStr(value_buf, 32, value); @@ -375,15 +404,20 @@ Status Redis::HIncrbyfloat(const Slice& key, const Slice& field, const Slice& by } BaseMetaKey base_meta_key(key); - Status s = db_->Get(default_read_options_, handles_[kHashesMetaCF], base_meta_key.Encode(), &meta_value); + Status s = db_->Get(default_read_options_, handles_[kMetaCF], base_meta_key.Encode(), &meta_value); char meta_value_buf[4] = {0}; if (s.ok()) { + auto type = static_cast(static_cast(meta_value[0])); + if (type != Type::kHash) { + return Status::InvalidArgument("WRONGTYPE Operation against a key holding the wrong kind of value"); + } ParsedHashesMetaValue parsed_hashes_meta_value(&meta_value); + if (parsed_hashes_meta_value.IsStale() || parsed_hashes_meta_value.Count() == 0) { version = parsed_hashes_meta_value.UpdateVersion(); parsed_hashes_meta_value.SetCount(1); parsed_hashes_meta_value.SetEtime(0); - batch.Put(handles_[kHashesMetaCF], base_meta_key.Encode(), meta_value); + batch.Put(handles_[kMetaCF], base_meta_key.Encode(), meta_value); HashesDataKey hashes_data_key(key, version, field); LongDoubleToStr(long_double_by, new_value); @@ -416,7 +450,7 @@ Status Redis::HIncrbyfloat(const Slice& key, const Slice& field, const Slice& by } parsed_hashes_meta_value.ModifyCount(1); BaseDataValue internal_value(*new_value); - batch.Put(handles_[kHashesMetaCF], base_meta_key.Encode(), meta_value); + batch.Put(handles_[kMetaCF], base_meta_key.Encode(), meta_value); batch.Put(handles_[kHashesDataCF], hashes_data_key.Encode(), internal_value.Encode()); } else { return s; @@ -424,9 +458,9 @@ Status Redis::HIncrbyfloat(const Slice& key, const Slice& field, const Slice& by } } else if (s.IsNotFound()) { EncodeFixed32(meta_value_buf, 1); - HashesMetaValue hashes_meta_value(Slice(meta_value_buf, sizeof(int32_t))); + HashesMetaValue hashes_meta_value(Type::kHash, Slice(meta_value_buf, 4)); version = hashes_meta_value.UpdateVersion(); - batch.Put(handles_[kHashesMetaCF], base_meta_key.Encode(), hashes_meta_value.Encode()); + batch.Put(handles_[kMetaCF], base_meta_key.Encode(), hashes_meta_value.Encode()); HashesDataKey hashes_data_key(key, version, field); LongDoubleToStr(long_double_by, new_value); @@ -450,9 +484,14 @@ Status Redis::HKeys(const Slice& key, std::vector* fields) { read_options.snapshot = snapshot; BaseMetaKey base_meta_key(key); - Status s = db_->Get(read_options, handles_[kHashesMetaCF], base_meta_key.Encode(), &meta_value); + Status s = db_->Get(read_options, handles_[kMetaCF], base_meta_key.Encode(), &meta_value); if (s.ok()) { + auto type = static_cast(static_cast(meta_value[0])); + if (type != Type::kHash) { + return Status::InvalidArgument("WRONGTYPE Operation against a key holding the wrong kind of value"); + } ParsedHashesMetaValue parsed_hashes_meta_value(&meta_value); + if (parsed_hashes_meta_value.IsStale()) { return Status::NotFound("Stale"); } else if (parsed_hashes_meta_value.Count() == 0) { @@ -478,9 +517,14 @@ Status Redis::HLen(const Slice& key, int32_t* ret) { std::string meta_value; BaseMetaKey base_meta_key(key); - Status s = db_->Get(default_read_options_, handles_[kHashesMetaCF], base_meta_key.Encode(), &meta_value); + Status s = db_->Get(default_read_options_, handles_[kMetaCF], base_meta_key.Encode(), &meta_value); if (s.ok()) { + auto type = static_cast(static_cast(meta_value[0])); + if (type != Type::kHash) { + return Status::InvalidArgument("WRONGTYPE Operation against a key holding the wrong kind of value"); + } ParsedHashesMetaValue parsed_hashes_meta_value(&meta_value); + if (parsed_hashes_meta_value.IsStale()) { *ret = 0; return Status::NotFound("Stale"); @@ -507,9 +551,14 @@ Status Redis::HMGet(const Slice& key, const std::vector& fields, st ScopeSnapshot ss(db_, &snapshot); read_options.snapshot = snapshot; BaseMetaKey base_meta_key(key); - Status s = db_->Get(read_options, handles_[kHashesMetaCF], base_meta_key.Encode(), &meta_value); + Status s = db_->Get(read_options, handles_[kMetaCF], base_meta_key.Encode(), &meta_value); if (s.ok()) { + auto type = static_cast(static_cast(meta_value[0])); + if (type != Type::kHash) { + return Status::InvalidArgument("WRONGTYPE Operation against a key holding the wrong kind of value"); + } ParsedHashesMetaValue parsed_hashes_meta_value(&meta_value); + if ((is_stale = parsed_hashes_meta_value.IsStale()) || parsed_hashes_meta_value.Count() == 0) { for (size_t idx = 0; idx < fields.size(); ++idx) { vss->push_back({std::string(), Status::NotFound()}); @@ -560,17 +609,22 @@ Status Redis::HMSet(const Slice& key, const std::vector& fvs) { std::string meta_value; BaseMetaKey base_meta_key(key); - Status s = db_->Get(default_read_options_, handles_[kHashesMetaCF], base_meta_key.Encode(), &meta_value); + Status s = db_->Get(default_read_options_, handles_[kMetaCF], base_meta_key.Encode(), &meta_value); char meta_value_buf[4] = {0}; if (s.ok()) { + auto type = static_cast(static_cast(meta_value[0])); + if (type != Type::kHash) { + return Status::InvalidArgument("WRONGTYPE Operation against a key holding the wrong kind of value"); + } ParsedHashesMetaValue parsed_hashes_meta_value(&meta_value); + if (parsed_hashes_meta_value.IsStale() || parsed_hashes_meta_value.Count() == 0) { version = parsed_hashes_meta_value.InitialMetaValue(); if (!parsed_hashes_meta_value.check_set_count(static_cast(filtered_fvs.size()))) { return Status::InvalidArgument("hash size overflow"); } parsed_hashes_meta_value.SetCount(static_cast(filtered_fvs.size())); - batch.Put(handles_[kHashesMetaCF], base_meta_key.Encode(), meta_value); + batch.Put(handles_[kMetaCF], base_meta_key.Encode(), meta_value); for (const auto& fv : filtered_fvs) { HashesDataKey hashes_data_key(key, version, fv.field); BaseDataValue inter_value(fv.value); @@ -598,13 +652,13 @@ Status Redis::HMSet(const Slice& key, const std::vector& fvs) { return Status::InvalidArgument("hash size overflow"); } parsed_hashes_meta_value.ModifyCount(count); - batch.Put(handles_[kHashesMetaCF], base_meta_key.Encode(), meta_value); + batch.Put(handles_[kMetaCF], base_meta_key.Encode(), meta_value); } } else if (s.IsNotFound()) { EncodeFixed32(meta_value_buf, filtered_fvs.size()); - HashesMetaValue hashes_meta_value(Slice(meta_value_buf, sizeof(int32_t))); + HashesMetaValue hashes_meta_value(Type::kHash, Slice(meta_value_buf, 4)); version = hashes_meta_value.UpdateVersion(); - batch.Put(handles_[kHashesMetaCF], base_meta_key.Encode(), hashes_meta_value.Encode()); + batch.Put(handles_[kMetaCF], base_meta_key.Encode(), hashes_meta_value.Encode()); for (const auto& fv : filtered_fvs) { HashesDataKey hashes_data_key(key, version, fv.field); BaseDataValue inter_value(fv.value); @@ -625,14 +679,22 @@ Status Redis::HSet(const Slice& key, const Slice& field, const Slice& value, int std::string meta_value; BaseMetaKey base_meta_key(key); - Status s = db_->Get(default_read_options_, handles_[kHashesMetaCF], base_meta_key.Encode(), &meta_value); + Status s = db_->Get(default_read_options_, handles_[kMetaCF], base_meta_key.Encode(), &meta_value); char meta_value_buf[4] = {0}; if (s.ok()) { + auto type = static_cast(static_cast(meta_value[0])); + if (type != Type::kHash) { + return Status::InvalidArgument("WRONGTYPE Operation against a key holding the wrong kind of value"); + } ParsedHashesMetaValue parsed_hashes_meta_value(&meta_value); + + if (!parsed_hashes_meta_value.IsSameType(Type::kHash)) { + return Status::InvalidArgument("WRONGTYPE Operation against a key holding the wrong kind of value"); + } if (parsed_hashes_meta_value.IsStale() || parsed_hashes_meta_value.Count() == 0) { version = parsed_hashes_meta_value.InitialMetaValue(); parsed_hashes_meta_value.SetCount(1); - batch->Put(kHashesMetaCF, base_meta_key.Encode(), meta_value); + batch->Put(kMetaCF, base_meta_key.Encode(), meta_value); HashesDataKey data_key(key, version, field); BaseDataValue internal_value(value); batch->Put(kHashesDataCF, data_key.Encode(), internal_value.Encode()); @@ -657,8 +719,9 @@ Status Redis::HSet(const Slice& key, const Slice& field, const Slice& value, int } parsed_hashes_meta_value.ModifyCount(1); BaseDataValue internal_value(value); - batch->Put(kHashesMetaCF, base_meta_key.Encode(), meta_value); + batch->Put(kMetaCF, base_meta_key.Encode(), meta_value); batch->Put(kHashesDataCF, hashes_data_key.Encode(), internal_value.Encode()); + *res = 1; } else { return s; @@ -666,9 +729,9 @@ Status Redis::HSet(const Slice& key, const Slice& field, const Slice& value, int } } else if (s.IsNotFound()) { EncodeFixed32(meta_value_buf, 1); - HashesMetaValue meta_value(Slice(meta_value_buf, sizeof(int32_t))); - version = meta_value.UpdateVersion(); - batch->Put(kHashesMetaCF, base_meta_key.Encode(), meta_value.Encode()); + HashesMetaValue hashes_meta_value(Type::kHash, Slice(meta_value_buf, 4)); + version = hashes_meta_value.UpdateVersion(); + batch->Put(kMetaCF, base_meta_key.Encode(), hashes_meta_value.Encode()); HashesDataKey data_key(key, version, field); BaseDataValue internal_value(value); batch->Put(kHashesDataCF, data_key.Encode(), internal_value.Encode()); @@ -690,14 +753,19 @@ Status Redis::HSetnx(const Slice& key, const Slice& field, const Slice& value, i BaseMetaKey base_meta_key(key); BaseDataValue internal_value(value); - Status s = db_->Get(default_read_options_, handles_[kHashesMetaCF], base_meta_key.Encode(), &meta_value); + Status s = db_->Get(default_read_options_, handles_[kMetaCF], base_meta_key.Encode(), &meta_value); char meta_value_buf[4] = {0}; if (s.ok()) { + auto type = static_cast(static_cast(meta_value[0])); + if (type != Type::kHash) { + return Status::InvalidArgument("WRONGTYPE Operation against a key holding the wrong kind of value"); + } ParsedHashesMetaValue parsed_hashes_meta_value(&meta_value); + if (parsed_hashes_meta_value.IsStale() || parsed_hashes_meta_value.Count() == 0) { version = parsed_hashes_meta_value.InitialMetaValue(); parsed_hashes_meta_value.SetCount(1); - batch.Put(handles_[kHashesMetaCF], base_meta_key.Encode(), meta_value); + batch.Put(handles_[kMetaCF], base_meta_key.Encode(), meta_value); HashesDataKey hashes_data_key(key, version, field); batch.Put(handles_[kHashesDataCF], hashes_data_key.Encode(), internal_value.Encode()); *ret = 1; @@ -713,7 +781,7 @@ Status Redis::HSetnx(const Slice& key, const Slice& field, const Slice& value, i return Status::InvalidArgument("hash size overflow"); } parsed_hashes_meta_value.ModifyCount(1); - batch.Put(handles_[kHashesMetaCF], base_meta_key.Encode(), meta_value); + batch.Put(handles_[kMetaCF], base_meta_key.Encode(), meta_value); batch.Put(handles_[kHashesDataCF], hashes_data_key.Encode(), internal_value.Encode()); *ret = 1; } else { @@ -722,9 +790,9 @@ Status Redis::HSetnx(const Slice& key, const Slice& field, const Slice& value, i } } else if (s.IsNotFound()) { EncodeFixed32(meta_value_buf, 1); - HashesMetaValue hashes_meta_value(Slice(meta_value_buf, sizeof(int32_t))); + HashesMetaValue hashes_meta_value(Type::kHash, Slice(meta_value_buf, 4)); version = hashes_meta_value.UpdateVersion(); - batch.Put(handles_[kHashesMetaCF], base_meta_key.Encode(), hashes_meta_value.Encode()); + batch.Put(handles_[kMetaCF], base_meta_key.Encode(), hashes_meta_value.Encode()); HashesDataKey hashes_data_key(key, version, field); batch.Put(handles_[kHashesDataCF], hashes_data_key.Encode(), internal_value.Encode()); *ret = 1; @@ -744,9 +812,14 @@ Status Redis::HVals(const Slice& key, std::vector* values) { read_options.snapshot = snapshot; BaseMetaKey base_meta_key(key); - Status s = db_->Get(read_options, handles_[kHashesMetaCF], base_meta_key.Encode(), &meta_value); + Status s = db_->Get(read_options, handles_[kMetaCF], base_meta_key.Encode(), &meta_value); if (s.ok()) { + auto type = static_cast(static_cast(meta_value[0])); + if (type != Type::kHash) { + return Status::InvalidArgument("WRONGTYPE Operation against a key holding the wrong kind of value"); + } ParsedHashesMetaValue parsed_hashes_meta_value(&meta_value); + if (parsed_hashes_meta_value.IsStale()) { return Status::NotFound("Stale"); } else if (parsed_hashes_meta_value.Count() == 0) { @@ -797,9 +870,14 @@ Status Redis::HScan(const Slice& key, int64_t cursor, const std::string& pattern read_options.snapshot = snapshot; BaseMetaKey base_meta_key(key); - Status s = db_->Get(read_options, handles_[kHashesMetaCF], base_meta_key.Encode(), &meta_value); + Status s = db_->Get(read_options, handles_[kMetaCF], base_meta_key.Encode(), &meta_value); if (s.ok()) { + auto type = static_cast(static_cast(meta_value[0])); + if (type != Type::kHash) { + return Status::InvalidArgument("WRONGTYPE Operation against a key holding the wrong kind of value"); + } ParsedHashesMetaValue parsed_hashes_meta_value(&meta_value); + if (parsed_hashes_meta_value.IsStale() || parsed_hashes_meta_value.Count() == 0) { *next_cursor = 0; return Status::NotFound(); @@ -864,9 +942,14 @@ Status Redis::HScanx(const Slice& key, const std::string& start_field, const std read_options.snapshot = snapshot; BaseMetaKey base_meta_key(key); - Status s = db_->Get(read_options, handles_[kHashesMetaCF], base_meta_key.Encode(), &meta_value); + Status s = db_->Get(read_options, handles_[kMetaCF], base_meta_key.Encode(), &meta_value); if (s.ok()) { + auto type = static_cast(static_cast(meta_value[0])); + if (type != Type::kHash) { + return Status::InvalidArgument("WRONGTYPE Operation against a key holding the wrong kind of value"); + } ParsedHashesMetaValue parsed_hashes_meta_value(&meta_value); + if (parsed_hashes_meta_value.IsStale() || parsed_hashes_meta_value.Count() == 0) { *next_field = ""; return Status::NotFound(); @@ -906,11 +989,14 @@ Status Redis::HScanx(const Slice& key, const std::string& start_field, const std Status Redis::HRandField(const Slice& key, int64_t count, bool with_values, std::vector* res) { BaseMetaKey base_meta_key(key); std::string meta_value; - Status s = db_->Get(default_read_options_, handles_[kHashesMetaCF], base_meta_key.Encode(), &meta_value); + Status s = db_->Get(default_read_options_, handles_[kMetaCF], base_meta_key.Encode(), &meta_value); if (!s.ok()) { return s; } ParsedHashesMetaValue parsed_hashes_meta_value(&meta_value); + if (!parsed_hashes_meta_value.IsSameType(Type::kHash)) { + return Status::InvalidArgument("WRONGTYPE Operation against a key holding the wrong kind of value"); + } auto hlen = parsed_hashes_meta_value.Count(); if (parsed_hashes_meta_value.IsStale() || hlen == 0) { return Status::NotFound(); @@ -997,9 +1083,14 @@ Status Redis::PKHScanRange(const Slice& key, const Slice& field_start, const std } BaseMetaKey base_meta_key(key); - Status s = db_->Get(read_options, handles_[kHashesMetaCF], base_meta_key.Encode(), &meta_value); + Status s = db_->Get(read_options, handles_[kMetaCF], base_meta_key.Encode(), &meta_value); if (s.ok()) { + auto type = static_cast(static_cast(meta_value[0])); + if (type != Type::kHash) { + return Status::InvalidArgument("WRONGTYPE Operation against a key holding the wrong kind of value"); + } ParsedHashesMetaValue parsed_hashes_meta_value(&meta_value); + if (parsed_hashes_meta_value.IsStale() || parsed_hashes_meta_value.Count() == 0) { return Status::NotFound(); } else { @@ -1058,9 +1149,14 @@ Status Redis::PKHRScanRange(const Slice& key, const Slice& field_start, const st } BaseMetaKey base_meta_key(key); - Status s = db_->Get(read_options, handles_[kHashesMetaCF], base_meta_key.Encode(), &meta_value); + Status s = db_->Get(read_options, handles_[kMetaCF], base_meta_key.Encode(), &meta_value); if (s.ok()) { + auto type = static_cast(static_cast(meta_value[0])); + if (type != Type::kHash) { + return Status::InvalidArgument("WRONGTYPE Operation against a key holding the wrong kind of value"); + } ParsedHashesMetaValue parsed_hashes_meta_value(&meta_value); + if (parsed_hashes_meta_value.IsStale() || parsed_hashes_meta_value.Count() == 0) { return Status::NotFound(); } else { @@ -1105,9 +1201,14 @@ Status Redis::HashesExpire(const Slice& key, uint64_t ttl) { ScopeRecordLock l(lock_mgr_, key); BaseMetaKey base_meta_key(key); - Status s = db_->Get(default_read_options_, handles_[kHashesMetaCF], base_meta_key.Encode(), &meta_value); + Status s = db_->Get(default_read_options_, handles_[kMetaCF], base_meta_key.Encode(), &meta_value); if (s.ok()) { + auto type = static_cast(static_cast(meta_value[0])); + if (type != Type::kHash) { + return Status::InvalidArgument("WRONGTYPE Operation against a key holding the wrong kind of value"); + } ParsedHashesMetaValue parsed_hashes_meta_value(&meta_value); + if (parsed_hashes_meta_value.IsStale()) { return Status::NotFound("Stale"); } else if (parsed_hashes_meta_value.Count() == 0) { @@ -1116,10 +1217,10 @@ Status Redis::HashesExpire(const Slice& key, uint64_t ttl) { if (ttl > 0) { parsed_hashes_meta_value.SetRelativeTimestamp(ttl); - s = db_->Put(default_write_options_, handles_[kHashesMetaCF], base_meta_key.Encode(), meta_value); + s = db_->Put(default_write_options_, handles_[kMetaCF], base_meta_key.Encode(), meta_value); } else { parsed_hashes_meta_value.InitialMetaValue(); - s = db_->Put(default_write_options_, handles_[kHashesMetaCF], base_meta_key.Encode(), meta_value); + s = db_->Put(default_write_options_, handles_[kMetaCF], base_meta_key.Encode(), meta_value); } } return s; @@ -1130,9 +1231,14 @@ Status Redis::HashesDel(const Slice& key) { ScopeRecordLock l(lock_mgr_, key); BaseMetaKey base_meta_key(key); - Status s = db_->Get(default_read_options_, handles_[kHashesMetaCF], base_meta_key.Encode(), &meta_value); + Status s = db_->Get(default_read_options_, handles_[kMetaCF], base_meta_key.Encode(), &meta_value); if (s.ok()) { + auto type = static_cast(static_cast(meta_value[0])); + if (type != Type::kHash) { + return Status::InvalidArgument("WRONGTYPE Operation against a key holding the wrong kind of value"); + } ParsedHashesMetaValue parsed_hashes_meta_value(&meta_value); + if (parsed_hashes_meta_value.IsStale()) { return Status::NotFound("Stale"); } else if (parsed_hashes_meta_value.Count() == 0) { @@ -1140,7 +1246,7 @@ Status Redis::HashesDel(const Slice& key) { } else { uint32_t statistic = parsed_hashes_meta_value.Count(); parsed_hashes_meta_value.InitialMetaValue(); - s = db_->Put(default_write_options_, handles_[kHashesMetaCF], base_meta_key.Encode(), meta_value); + s = db_->Put(default_write_options_, handles_[kMetaCF], base_meta_key.Encode(), meta_value); UpdateSpecificKeyStatistics(DataType::kHashes, key.ToString(), statistic); } } @@ -1152,9 +1258,14 @@ Status Redis::HashesExpireat(const Slice& key, uint64_t timestamp) { ScopeRecordLock l(lock_mgr_, key); BaseMetaKey base_meta_key(key); - Status s = db_->Get(default_read_options_, handles_[kHashesMetaCF], base_meta_key.Encode(), &meta_value); + Status s = db_->Get(default_read_options_, handles_[kMetaCF], base_meta_key.Encode(), &meta_value); if (s.ok()) { + auto type = static_cast(static_cast(meta_value[0])); + if (type != Type::kHash) { + return Status::InvalidArgument("WRONGTYPE Operation against a key holding the wrong kind of value"); + } ParsedHashesMetaValue parsed_hashes_meta_value(&meta_value); + if (parsed_hashes_meta_value.IsStale()) { return Status::NotFound("Stale"); } else if (parsed_hashes_meta_value.Count() == 0) { @@ -1165,7 +1276,7 @@ Status Redis::HashesExpireat(const Slice& key, uint64_t timestamp) { } else { parsed_hashes_meta_value.InitialMetaValue(); } - s = db_->Put(default_write_options_, handles_[kHashesMetaCF], base_meta_key.Encode(), meta_value); + s = db_->Put(default_write_options_, handles_[kMetaCF], base_meta_key.Encode(), meta_value); } } return s; @@ -1176,9 +1287,14 @@ Status Redis::HashesPersist(const Slice& key) { ScopeRecordLock l(lock_mgr_, key); BaseMetaKey base_meta_key(key); - Status s = db_->Get(default_read_options_, handles_[kHashesMetaCF], base_meta_key.Encode(), &meta_value); + Status s = db_->Get(default_read_options_, handles_[kMetaCF], base_meta_key.Encode(), &meta_value); if (s.ok()) { + auto type = static_cast(static_cast(meta_value[0])); + if (type != Type::kHash) { + return Status::InvalidArgument("WRONGTYPE Operation against a key holding the wrong kind of value"); + } ParsedHashesMetaValue parsed_hashes_meta_value(&meta_value); + if (parsed_hashes_meta_value.IsStale()) { return Status::NotFound("Stale"); } else if (parsed_hashes_meta_value.Count() == 0) { @@ -1189,7 +1305,7 @@ Status Redis::HashesPersist(const Slice& key) { return Status::NotFound("Not have an associated timeout"); } else { parsed_hashes_meta_value.SetEtime(0); - s = db_->Put(default_write_options_, handles_[kHashesMetaCF], base_meta_key.Encode(), meta_value); + s = db_->Put(default_write_options_, handles_[kMetaCF], base_meta_key.Encode(), meta_value); } } } @@ -1200,9 +1316,14 @@ Status Redis::HashesTTL(const Slice& key, uint64_t* timestamp) { std::string meta_value; BaseMetaKey base_meta_key(key); - Status s = db_->Get(default_read_options_, handles_[kHashesMetaCF], base_meta_key.Encode(), &meta_value); + Status s = db_->Get(default_read_options_, handles_[kMetaCF], base_meta_key.Encode(), &meta_value); if (s.ok()) { + auto type = static_cast(static_cast(meta_value[0])); + if (type != Type::kHash) { + return Status::InvalidArgument("WRONGTYPE Operation against a key holding the wrong kind of value"); + } ParsedHashesMetaValue parsed_hashes_meta_value(&meta_value); + if (parsed_hashes_meta_value.IsStale()) { *timestamp = -2; return Status::NotFound("Stale"); @@ -1234,7 +1355,7 @@ Status Redis::HashesRename(const Slice& key, Redis* new_inst, const Slice& newke BaseMetaKey base_meta_key(key); BaseMetaKey base_meta_newkey(newkey); - s = db_->Get(default_read_options_, handles_[kHashesMetaCF], base_meta_key.Encode(), &meta_value); + s = db_->Get(default_read_options_, handles_[kMetaCF], base_meta_key.Encode(), &meta_value); if (s.ok()) { ParsedHashesMetaValue parsed_hashes_meta_value(&meta_value); if (parsed_hashes_meta_value.IsStale()) { @@ -1244,12 +1365,12 @@ Status Redis::HashesRename(const Slice& key, Redis* new_inst, const Slice& newke } // copy a new hash with newkey statistic = parsed_hashes_meta_value.Count(); - s = new_inst->GetDB()->Put(default_write_options_, handles_[kHashesMetaCF], base_meta_newkey.Encode(), meta_value); + s = new_inst->GetDB()->Put(default_write_options_, handles_[kMetaCF], base_meta_newkey.Encode(), meta_value); new_inst->UpdateSpecificKeyStatistics(DataType::kHashes, newkey.ToString(), statistic); // HashesDel key parsed_hashes_meta_value.InitialMetaValue(); - s = db_->Put(default_write_options_, handles_[kHashesMetaCF], base_meta_key.Encode(), meta_value); + s = db_->Put(default_write_options_, handles_[kMetaCF], base_meta_key.Encode(), meta_value); UpdateSpecificKeyStatistics(DataType::kHashes, key.ToString(), statistic); } return s; @@ -1264,7 +1385,7 @@ Status Redis::HashesRenamenx(const Slice& key, Redis* new_inst, const Slice& new BaseMetaKey base_meta_key(key); BaseMetaKey base_meta_newkey(newkey); - s = db_->Get(default_read_options_, handles_[kHashesMetaCF], base_meta_key.Encode(), &meta_value); + s = db_->Get(default_read_options_, handles_[kMetaCF], base_meta_key.Encode(), &meta_value); if (s.ok()) { ParsedHashesMetaValue parsed_hashes_meta_value(&meta_value); if (parsed_hashes_meta_value.IsStale()) { @@ -1274,7 +1395,7 @@ Status Redis::HashesRenamenx(const Slice& key, Redis* new_inst, const Slice& new } // check if newkey exists. std::string new_meta_value; - s = new_inst->GetDB()->Get(default_read_options_, handles_[kHashesMetaCF], base_meta_newkey.Encode(), + s = new_inst->GetDB()->Get(default_read_options_, handles_[kMetaCF], base_meta_newkey.Encode(), &new_meta_value); if (s.ok()) { ParsedHashesMetaValue parsed_hashes_new_meta_value(&new_meta_value); @@ -1285,12 +1406,12 @@ Status Redis::HashesRenamenx(const Slice& key, Redis* new_inst, const Slice& new // copy a new hash with newkey statistic = parsed_hashes_meta_value.Count(); - s = new_inst->GetDB()->Put(default_write_options_, handles_[kHashesMetaCF], base_meta_newkey.Encode(), meta_value); + s = new_inst->GetDB()->Put(default_write_options_, handles_[kMetaCF], base_meta_newkey.Encode(), meta_value); new_inst->UpdateSpecificKeyStatistics(DataType::kHashes, newkey.ToString(), statistic); // HashesDel key parsed_hashes_meta_value.InitialMetaValue(); - s = db_->Put(default_write_options_, handles_[kHashesMetaCF], base_meta_key.Encode(), meta_value); + s = db_->Put(default_write_options_, handles_[kMetaCF], base_meta_key.Encode(), meta_value); UpdateSpecificKeyStatistics(DataType::kHashes, key.ToString(), statistic); } return s; @@ -1305,9 +1426,12 @@ void Redis::ScanHashes() { auto current_time = static_cast(time(nullptr)); INFO("***************rocksdb instance: {} Hashes Meta Data***************", index_); - auto meta_iter = db_->NewIterator(iterator_options, handles_[kHashesMetaCF]); + auto meta_iter = db_->NewIterator(iterator_options, handles_[kMetaCF]); for (meta_iter->SeekToFirst(); meta_iter->Valid(); meta_iter->Next()) { ParsedHashesMetaValue parsed_hashes_meta_value(meta_iter->value()); + if (!parsed_hashes_meta_value.IsSameType(Type::kHash)) { + continue; + } uint64_t survival_time = 0; if (parsed_hashes_meta_value.Etime() != 0) { survival_time = diff --git a/src/storage/src/redis_lists.cc b/src/storage/src/redis_lists.cc index db2b15b05..8263a29cd 100644 --- a/src/storage/src/redis_lists.cc +++ b/src/storage/src/redis_lists.cc @@ -31,9 +31,12 @@ Status Redis::ScanListsKeyNum(KeyInfo* key_info) { int64_t curtime; rocksdb::Env::Default()->GetCurrentTime(&curtime); - rocksdb::Iterator* iter = db_->NewIterator(iterator_options, handles_[kListsMetaCF]); + rocksdb::Iterator* iter = db_->NewIterator(iterator_options, handles_[kMetaCF]); for (iter->SeekToFirst(); iter->Valid(); iter->Next()) { ParsedListsMetaValue parsed_lists_meta_value(iter->value()); + if (!parsed_lists_meta_value.IsSameType(Type::kList)) { + continue; + } if (parsed_lists_meta_value.IsStale() || parsed_lists_meta_value.Count() == 0) { invaild_keys++; } else { @@ -65,17 +68,20 @@ Status Redis::ListsPKPatternMatchDel(const std::string& pattern, int32_t* ret) { int32_t total_delete = 0; Status s; rocksdb::WriteBatch batch; - rocksdb::Iterator* iter = db_->NewIterator(iterator_options, handles_[kListsMetaCF]); + rocksdb::Iterator* iter = db_->NewIterator(iterator_options, handles_[kMetaCF]); iter->SeekToFirst(); while (iter->Valid()) { ParsedBaseMetaKey parsed_meta_key(iter->key().ToString()); meta_value = iter->value().ToString(); ParsedListsMetaValue parsed_lists_meta_value(&meta_value); + if (!parsed_lists_meta_value.IsSameType(Type::kList)) { + continue; + } if (!parsed_lists_meta_value.IsStale() && (parsed_lists_meta_value.Count() != 0U) && (StringMatch(pattern.data(), pattern.size(), parsed_meta_key.Key().data(), parsed_meta_key.Key().size(), 0) != 0)) { parsed_lists_meta_value.InitialMetaValue(); - batch.Put(handles_[kListsMetaCF], iter->key(), meta_value); + batch.Put(handles_[kMetaCF], iter->key(), meta_value); } if (static_cast(batch.Count()) >= BATCH_DELETE_LIMIT) { s = db_->Write(default_write_options_, &batch); @@ -110,8 +116,12 @@ Status Redis::LIndex(const Slice& key, int64_t index, std::string* element) { std::string meta_value; BaseMetaKey base_meta_key(key); - Status s = db_->Get(read_options, handles_[kListsMetaCF], base_meta_key.Encode(), &meta_value); + Status s = db_->Get(read_options, handles_[kMetaCF], base_meta_key.Encode(), &meta_value); if (s.ok()) { + auto type = static_cast(static_cast(meta_value[0])); + if (type != Type::kList) { + return Status::InvalidArgument("WRONGTYPE Operation against a key holding the wrong kind of value"); + } ParsedListsMetaValue parsed_lists_meta_value(&meta_value); uint64_t version = parsed_lists_meta_value.Version(); if (parsed_lists_meta_value.IsStale()) { @@ -144,8 +154,12 @@ Status Redis::LInsert(const Slice& key, const BeforeOrAfter& before_or_after, co std::string meta_value; BaseMetaKey base_meta_key(key); - Status s = db_->Get(default_read_options_, handles_[kListsMetaCF], base_meta_key.Encode(), &meta_value); + Status s = db_->Get(default_read_options_, handles_[kMetaCF], base_meta_key.Encode(), &meta_value); if (s.ok()) { + auto type = static_cast(static_cast(meta_value[0])); + if (type != Type::kList) { + return Status::InvalidArgument("WRONGTYPE Operation against a key holding the wrong kind of value"); + } ParsedListsMetaValue parsed_lists_meta_value(&meta_value); if (parsed_lists_meta_value.IsStale()) { return Status::NotFound("Stale"); @@ -226,7 +240,7 @@ Status Redis::LInsert(const Slice& key, const BeforeOrAfter& before_or_after, co parsed_lists_meta_value.ModifyRightIndex(1); } parsed_lists_meta_value.ModifyCount(1); - batch->Put(kListsMetaCF, base_meta_key.Encode(), meta_value); + batch->Put(kMetaCF, base_meta_key.Encode(), meta_value); ListsDataKey lists_target_key(key, version, target_index); BaseDataValue i_val(value); batch->Put(kListsDataCF, lists_target_key.Encode(), i_val.Encode()); @@ -245,8 +259,12 @@ Status Redis::LLen(const Slice& key, uint64_t* len) { std::string meta_value; BaseMetaKey base_meta_key(key); - Status s = db_->Get(default_read_options_, handles_[kListsMetaCF], base_meta_key.Encode(), &meta_value); + Status s = db_->Get(default_read_options_, handles_[kMetaCF], base_meta_key.Encode(), &meta_value); if (s.ok()) { + auto type = static_cast(static_cast(meta_value[0])); + if (type != Type::kList) { + return Status::InvalidArgument("WRONGTYPE Operation against a key holding the wrong kind of value"); + } ParsedListsMetaValue parsed_lists_meta_value(&meta_value); if (parsed_lists_meta_value.IsStale()) { return Status::NotFound("Stale"); @@ -270,8 +288,12 @@ Status Redis::LPop(const Slice& key, int64_t count, std::vector* el std::string meta_value; BaseMetaKey base_meta_key(key); - Status s = db_->Get(default_read_options_, handles_[kListsMetaCF], base_meta_key.Encode(), &meta_value); + Status s = db_->Get(default_read_options_, handles_[kMetaCF], base_meta_key.Encode(), &meta_value); if (s.ok()) { + auto type = static_cast(static_cast(meta_value[0])); + if (type != Type::kList) { + return Status::InvalidArgument("WRONGTYPE Operation against a key holding the wrong kind of value"); + } ParsedListsMetaValue parsed_lists_meta_value(&meta_value); if (parsed_lists_meta_value.IsStale()) { return Status::NotFound("Stale"); @@ -294,7 +316,7 @@ Status Redis::LPop(const Slice& key, int64_t count, std::vector* el parsed_lists_meta_value.ModifyCount(-1); parsed_lists_meta_value.ModifyLeftIndex(-1); } - batch->Put(kListsMetaCF, base_meta_key.Encode(), meta_value); + batch->Put(kMetaCF, base_meta_key.Encode(), meta_value); delete iter; } } @@ -315,8 +337,12 @@ Status Redis::LPush(const Slice& key, const std::vector& values, ui std::string meta_value; BaseMetaKey base_meta_key(key); - Status s = db_->Get(default_read_options_, handles_[kListsMetaCF], base_meta_key.Encode(), &meta_value); + Status s = db_->Get(default_read_options_, handles_[kMetaCF], base_meta_key.Encode(), &meta_value); if (s.ok()) { + auto type = static_cast(static_cast(meta_value[0])); + if (type != Type::kList) { + return Status::InvalidArgument("WRONGTYPE Operation against a key holding the wrong kind of value"); + } ParsedListsMetaValue parsed_lists_meta_value(&meta_value); if (parsed_lists_meta_value.IsStale() || parsed_lists_meta_value.Count() == 0) { version = parsed_lists_meta_value.InitialMetaValue(); @@ -331,12 +357,12 @@ Status Redis::LPush(const Slice& key, const std::vector& values, ui BaseDataValue i_val(value); batch->Put(kListsDataCF, lists_data_key.Encode(), i_val.Encode()); } - batch->Put(kListsMetaCF, base_meta_key.Encode(), meta_value); + batch->Put(kMetaCF, base_meta_key.Encode(), meta_value); *ret = parsed_lists_meta_value.Count(); } else if (s.IsNotFound()) { char str[8]; EncodeFixed64(str, values.size()); - ListsMetaValue lists_meta_value(Slice(str, sizeof(uint64_t))); + ListsMetaValue lists_meta_value(Slice(str, 8)); version = lists_meta_value.UpdateVersion(); for (const auto& value : values) { index = lists_meta_value.LeftIndex(); @@ -345,7 +371,7 @@ Status Redis::LPush(const Slice& key, const std::vector& values, ui BaseDataValue i_val(value); batch->Put(kListsDataCF, lists_data_key.Encode(), i_val.Encode()); } - batch->Put(kListsMetaCF, base_meta_key.Encode(), lists_meta_value.Encode()); + batch->Put(kMetaCF, base_meta_key.Encode(), lists_meta_value.Encode()); *ret = lists_meta_value.RightIndex() - lists_meta_value.LeftIndex() - 1; } else { return s; @@ -361,9 +387,14 @@ Status Redis::LPushx(const Slice& key, const std::vector& values, u std::string meta_value; BaseMetaKey base_meta_key(key); - Status s = db_->Get(default_read_options_, handles_[kListsMetaCF], base_meta_key.Encode(), &meta_value); + Status s = db_->Get(default_read_options_, handles_[kMetaCF], base_meta_key.Encode(), &meta_value); if (s.ok()) { + auto type = static_cast(static_cast(meta_value[0])); + if (type != Type::kList) { + return Status::InvalidArgument("WRONGTYPE Operation against a key holding the wrong kind of value"); + } ParsedListsMetaValue parsed_lists_meta_value(&meta_value); + if (parsed_lists_meta_value.IsStale()) { return Status::NotFound("Stale"); } else if (parsed_lists_meta_value.Count() == 0) { @@ -378,7 +409,7 @@ Status Redis::LPushx(const Slice& key, const std::vector& values, u BaseDataValue i_val(value); batch->Put(kListsDataCF, lists_data_key.Encode(), i_val.Encode()); } - batch->Put(kListsMetaCF, base_meta_key.Encode(), meta_value); + batch->Put(kMetaCF, base_meta_key.Encode(), meta_value); *len = parsed_lists_meta_value.Count(); return batch->Commit(); } @@ -395,9 +426,14 @@ Status Redis::LRange(const Slice& key, int64_t start, int64_t stop, std::vector< std::string meta_value; BaseMetaKey base_meta_key(key); - Status s = db_->Get(read_options, handles_[kListsMetaCF], base_meta_key.Encode(), &meta_value); + Status s = db_->Get(read_options, handles_[kMetaCF], base_meta_key.Encode(), &meta_value); if (s.ok()) { + auto type = static_cast(static_cast(meta_value[0])); + if (type != Type::kList) { + return Status::InvalidArgument("WRONGTYPE Operation against a key holding the wrong kind of value"); + } ParsedListsMetaValue parsed_lists_meta_value(&meta_value); + if (parsed_lists_meta_value.IsStale()) { return Status::NotFound("Stale"); } else if (parsed_lists_meta_value.Count() == 0) { @@ -446,9 +482,14 @@ Status Redis::LRangeWithTTL(const Slice& key, int64_t start, int64_t stop, std:: std::string meta_value; BaseMetaKey base_meta_key(key); - Status s = db_->Get(read_options, handles_[kListsMetaCF], base_meta_key.Encode(), &meta_value); + Status s = db_->Get(read_options, handles_[kMetaCF], base_meta_key.Encode(), &meta_value); if (s.ok()) { + auto type = static_cast(static_cast(meta_value[0])); + if (type != Type::kList) { + return Status::InvalidArgument("WRONGTYPE Operation against a key holding the wrong kind of value"); + } ParsedListsMetaValue parsed_lists_meta_value(&meta_value); + if (parsed_lists_meta_value.Count() == 0) { return Status::NotFound(); } else if (parsed_lists_meta_value.IsStale()) { @@ -504,8 +545,12 @@ Status Redis::LRem(const Slice& key, int64_t count, const Slice& value, uint64_t std::string meta_value; BaseMetaKey base_meta_key(key); - Status s = db_->Get(default_read_options_, handles_[kListsMetaCF], base_meta_key.Encode(), &meta_value); + Status s = db_->Get(default_read_options_, handles_[kMetaCF], base_meta_key.Encode(), &meta_value); if (s.ok()) { + auto type = static_cast(static_cast(meta_value[0])); + if (type != Type::kList) { + return Status::InvalidArgument("WRONGTYPE Operation against a key holding the wrong kind of value"); + } ParsedListsMetaValue parsed_lists_meta_value(&meta_value); if (parsed_lists_meta_value.IsStale()) { return Status::NotFound("Stale"); @@ -605,7 +650,7 @@ Status Redis::LRem(const Slice& key, int64_t count, const Slice& value, uint64_t parsed_lists_meta_value.ModifyRightIndex(-target_index.size()); } parsed_lists_meta_value.ModifyCount(-target_index.size()); - batch->Put(kListsMetaCF, base_meta_key.Encode(), meta_value); + batch->Put(kMetaCF, base_meta_key.Encode(), meta_value); for (const auto& idx : delete_index) { ListsDataKey lists_data_key(key, version, idx); batch->Delete(kListsDataCF, lists_data_key.Encode()); @@ -627,9 +672,14 @@ Status Redis::LSet(const Slice& key, int64_t index, const Slice& value) { std::string meta_value; BaseMetaKey base_meta_key(key); - Status s = db_->Get(default_read_options_, handles_[kListsMetaCF], base_meta_key.Encode(), &meta_value); + Status s = db_->Get(default_read_options_, handles_[kMetaCF], base_meta_key.Encode(), &meta_value); if (s.ok()) { + auto type = static_cast(static_cast(meta_value[0])); + if (type != Type::kList) { + return Status::InvalidArgument("WRONGTYPE Operation against a key holding the wrong kind of value"); + } ParsedListsMetaValue parsed_lists_meta_value(&meta_value); + if (parsed_lists_meta_value.IsStale()) { return Status::NotFound("Stale"); } else if (parsed_lists_meta_value.Count() == 0) { @@ -660,9 +710,14 @@ Status Redis::LTrim(const Slice& key, int64_t start, int64_t stop) { std::string meta_value; BaseMetaKey base_meta_key(key); - Status s = db_->Get(default_read_options_, handles_[kListsMetaCF], base_meta_key.Encode(), &meta_value); + Status s = db_->Get(default_read_options_, handles_[kMetaCF], base_meta_key.Encode(), &meta_value); if (s.ok()) { + auto type = static_cast(static_cast(meta_value[0])); + if (type != Type::kList) { + return Status::InvalidArgument("WRONGTYPE Operation against a key holding the wrong kind of value"); + } ParsedListsMetaValue parsed_lists_meta_value(&meta_value); + uint64_t version = parsed_lists_meta_value.Version(); if (parsed_lists_meta_value.IsStale()) { return Status::NotFound("Stale"); @@ -677,7 +732,7 @@ Status Redis::LTrim(const Slice& key, int64_t start, int64_t stop) { if (sublist_left_index > sublist_right_index || sublist_left_index > origin_right_index || sublist_right_index < origin_left_index) { parsed_lists_meta_value.InitialMetaValue(); - batch->Put(kListsMetaCF, base_meta_key.Encode(), meta_value); + batch->Put(kMetaCF, base_meta_key.Encode(), meta_value); } else { if (sublist_left_index < origin_left_index) { sublist_left_index = origin_left_index; @@ -692,7 +747,7 @@ Status Redis::LTrim(const Slice& key, int64_t start, int64_t stop) { parsed_lists_meta_value.ModifyLeftIndex(-(sublist_left_index - origin_left_index)); parsed_lists_meta_value.ModifyRightIndex(-(origin_right_index - sublist_right_index)); parsed_lists_meta_value.ModifyCount(-delete_node_num); - batch->Put(kListsMetaCF, base_meta_key.Encode(), meta_value); + batch->Put(kMetaCF, base_meta_key.Encode(), meta_value); for (uint64_t idx = origin_left_index; idx < sublist_left_index; ++idx) { statistic++; ListsDataKey lists_data_key(key, version, idx); @@ -722,9 +777,14 @@ Status Redis::RPop(const Slice& key, int64_t count, std::vector* el std::string meta_value; BaseMetaKey base_meta_key(key); - Status s = db_->Get(default_read_options_, handles_[kListsMetaCF], base_meta_key.Encode(), &meta_value); + Status s = db_->Get(default_read_options_, handles_[kMetaCF], base_meta_key.Encode(), &meta_value); if (s.ok()) { + auto type = static_cast(static_cast(meta_value[0])); + if (type != Type::kList) { + return Status::InvalidArgument("WRONGTYPE Operation against a key holding the wrong kind of value"); + } ParsedListsMetaValue parsed_lists_meta_value(&meta_value); + if (parsed_lists_meta_value.IsStale()) { return Status::NotFound("Stale"); } else if (parsed_lists_meta_value.Count() == 0) { @@ -747,7 +807,7 @@ Status Redis::RPop(const Slice& key, int64_t count, std::vector* el parsed_lists_meta_value.ModifyCount(-1); parsed_lists_meta_value.ModifyRightIndex(-1); } - batch.Put(handles_[kListsMetaCF], base_meta_key.Encode(), meta_value); + batch.Put(handles_[kMetaCF], base_meta_key.Encode(), meta_value); delete iter; } } @@ -770,9 +830,14 @@ Status Redis::RPoplpush(const Slice& source, const Slice& destination, std::stri if (source.compare(destination) == 0) { std::string meta_value; BaseMetaKey base_source(source); - s = db_->Get(default_read_options_, handles_[kListsMetaCF], base_source.Encode(), &meta_value); + s = db_->Get(default_read_options_, handles_[kMetaCF], base_source.Encode(), &meta_value); if (s.ok()) { + auto type = static_cast(static_cast(meta_value[0])); + if (type != Type::kList) { + return Status::InvalidArgument("WRONGTYPE Operation against a key holding the wrong kind of value"); + } ParsedListsMetaValue parsed_lists_meta_value(&meta_value); + if (parsed_lists_meta_value.IsStale()) { return Status::NotFound("Stale"); } else if (parsed_lists_meta_value.Count() == 0) { @@ -797,7 +862,7 @@ Status Redis::RPoplpush(const Slice& source, const Slice& destination, std::stri statistic++; parsed_lists_meta_value.ModifyRightIndex(-1); parsed_lists_meta_value.ModifyLeftIndex(1); - batch.Put(handles_[kListsMetaCF], base_source.Encode(), meta_value); + batch.Put(handles_[kMetaCF], base_source.Encode(), meta_value); s = db_->Write(default_write_options_, &batch); UpdateSpecificKeyStatistics(DataType::kLists, source.ToString(), statistic); return s; @@ -815,9 +880,14 @@ Status Redis::RPoplpush(const Slice& source, const Slice& destination, std::stri std::string target; std::string source_meta_value; BaseMetaKey base_source(source); - s = db_->Get(default_read_options_, handles_[kListsMetaCF], base_source.Encode(), &source_meta_value); + s = db_->Get(default_read_options_, handles_[kMetaCF], base_source.Encode(), &source_meta_value); if (s.ok()) { + auto type = static_cast(static_cast(source_meta_value[0])); + if (type != Type::kList) { + return Status::InvalidArgument("WRONGTYPE Operation against a key holding the wrong kind of value"); + } ParsedListsMetaValue parsed_lists_meta_value(&source_meta_value); + if (parsed_lists_meta_value.IsStale()) { return Status::NotFound("Stale"); } else if (parsed_lists_meta_value.Count() == 0) { @@ -832,7 +902,7 @@ Status Redis::RPoplpush(const Slice& source, const Slice& destination, std::stri statistic++; parsed_lists_meta_value.ModifyCount(-1); parsed_lists_meta_value.ModifyRightIndex(-1); - batch.Put(handles_[kListsMetaCF], base_source.Encode(), source_meta_value); + batch.Put(handles_[kMetaCF], base_source.Encode(), source_meta_value); } else { return s; } @@ -843,9 +913,14 @@ Status Redis::RPoplpush(const Slice& source, const Slice& destination, std::stri std::string destination_meta_value; BaseMetaKey base_destination(destination); - s = db_->Get(default_read_options_, handles_[kListsMetaCF], base_destination.Encode(), &destination_meta_value); + s = db_->Get(default_read_options_, handles_[kMetaCF], base_destination.Encode(), &destination_meta_value); if (s.ok()) { + auto type = static_cast(static_cast(destination_meta_value[0])); + if (type != Type::kList) { + return Status::InvalidArgument("WRONGTYPE Operation against a key holding the wrong kind of value"); + } ParsedListsMetaValue parsed_lists_meta_value(&destination_meta_value); + if (parsed_lists_meta_value.IsStale() || parsed_lists_meta_value.Count() == 0) { version = parsed_lists_meta_value.InitialMetaValue(); } else { @@ -856,17 +931,17 @@ Status Redis::RPoplpush(const Slice& source, const Slice& destination, std::stri batch.Put(handles_[kListsDataCF], lists_data_key.Encode(), target); parsed_lists_meta_value.ModifyCount(1); parsed_lists_meta_value.ModifyLeftIndex(1); - batch.Put(handles_[kListsMetaCF], base_destination.Encode(), destination_meta_value); + batch.Put(handles_[kMetaCF], base_destination.Encode(), destination_meta_value); } else if (s.IsNotFound()) { char str[8]; EncodeFixed64(str, 1); - ListsMetaValue lists_meta_value(Slice(str, sizeof(uint64_t))); + ListsMetaValue lists_meta_value(Slice(str, 8)); version = lists_meta_value.UpdateVersion(); uint64_t target_index = lists_meta_value.LeftIndex(); ListsDataKey lists_data_key(destination, version, target_index); batch.Put(handles_[kListsDataCF], lists_data_key.Encode(), target); lists_meta_value.ModifyLeftIndex(1); - batch.Put(handles_[kListsMetaCF], base_destination.Encode(), lists_meta_value.Encode()); + batch.Put(handles_[kMetaCF], base_destination.Encode(), lists_meta_value.Encode()); } else { return s; } @@ -890,9 +965,14 @@ Status Redis::RPush(const Slice& key, const std::vector& values, ui std::string meta_value; BaseMetaKey base_meta_key(key); - Status s = db_->Get(default_read_options_, handles_[kListsMetaCF], base_meta_key.Encode(), &meta_value); + Status s = db_->Get(default_read_options_, handles_[kMetaCF], base_meta_key.Encode(), &meta_value); if (s.ok()) { + auto type = static_cast(static_cast(meta_value[0])); + if (type != Type::kList) { + return Status::InvalidArgument("WRONGTYPE Operation against a key holding the wrong kind of value"); + } ParsedListsMetaValue parsed_lists_meta_value(&meta_value); + if (parsed_lists_meta_value.IsStale() || parsed_lists_meta_value.Count() == 0) { version = parsed_lists_meta_value.InitialMetaValue(); } else { @@ -906,12 +986,12 @@ Status Redis::RPush(const Slice& key, const std::vector& values, ui BaseDataValue i_val(value); batch.Put(handles_[kListsDataCF], lists_data_key.Encode(), i_val.Encode()); } - batch.Put(handles_[kListsMetaCF], base_meta_key.Encode(), meta_value); + batch.Put(handles_[kMetaCF], base_meta_key.Encode(), meta_value); *ret = parsed_lists_meta_value.Count(); } else if (s.IsNotFound()) { char str[8]; EncodeFixed64(str, values.size()); - ListsMetaValue lists_meta_value(Slice(str, sizeof(uint64_t))); + ListsMetaValue lists_meta_value(Slice(str, 8)); version = lists_meta_value.UpdateVersion(); for (const auto& value : values) { index = lists_meta_value.RightIndex(); @@ -920,7 +1000,7 @@ Status Redis::RPush(const Slice& key, const std::vector& values, ui BaseDataValue i_val(value); batch.Put(handles_[kListsDataCF], lists_data_key.Encode(), i_val.Encode()); } - batch.Put(handles_[kListsMetaCF], base_meta_key.Encode(), lists_meta_value.Encode()); + batch.Put(handles_[kMetaCF], base_meta_key.Encode(), lists_meta_value.Encode()); *ret = lists_meta_value.RightIndex() - lists_meta_value.LeftIndex() - 1; } else { return s; @@ -936,9 +1016,14 @@ Status Redis::RPushx(const Slice& key, const std::vector& values, u std::string meta_value; BaseMetaKey base_meta_key(key); - Status s = db_->Get(default_read_options_, handles_[kListsMetaCF], base_meta_key.Encode(), &meta_value); + Status s = db_->Get(default_read_options_, handles_[kMetaCF], base_meta_key.Encode(), &meta_value); if (s.ok()) { + auto type = static_cast(static_cast(meta_value[0])); + if (type != Type::kList) { + return Status::InvalidArgument("WRONGTYPE Operation against a key holding the wrong kind of value"); + } ParsedListsMetaValue parsed_lists_meta_value(&meta_value); + if (parsed_lists_meta_value.IsStale()) { return Status::NotFound("Stale"); } else if (parsed_lists_meta_value.Count() == 0) { @@ -953,7 +1038,7 @@ Status Redis::RPushx(const Slice& key, const std::vector& values, u BaseDataValue i_val(value); batch.Put(handles_[kListsDataCF], lists_data_key.Encode(), i_val.Encode()); } - batch.Put(handles_[kListsMetaCF], base_meta_key.Encode(), meta_value); + batch.Put(handles_[kMetaCF], base_meta_key.Encode(), meta_value); *len = parsed_lists_meta_value.Count(); return db_->Write(default_write_options_, &batch); } @@ -966,9 +1051,14 @@ Status Redis::ListsExpire(const Slice& key, uint64_t ttl) { ScopeRecordLock l(lock_mgr_, key); BaseMetaKey base_meta_key(key); - Status s = db_->Get(default_read_options_, handles_[kListsMetaCF], base_meta_key.Encode(), &meta_value); + Status s = db_->Get(default_read_options_, handles_[kMetaCF], base_meta_key.Encode(), &meta_value); if (s.ok()) { + auto type = static_cast(static_cast(meta_value[0])); + if (type != Type::kList) { + return Status::InvalidArgument("WRONGTYPE Operation against a key holding the wrong kind of value"); + } ParsedListsMetaValue parsed_lists_meta_value(&meta_value); + if (parsed_lists_meta_value.IsStale()) { return Status::NotFound("Stale"); } else if (parsed_lists_meta_value.Count() == 0) { @@ -977,10 +1067,10 @@ Status Redis::ListsExpire(const Slice& key, uint64_t ttl) { if (ttl > 0) { parsed_lists_meta_value.SetRelativeTimestamp(ttl); - s = db_->Put(default_write_options_, handles_[kListsMetaCF], base_meta_key.Encode(), meta_value); + s = db_->Put(default_write_options_, handles_[kMetaCF], base_meta_key.Encode(), meta_value); } else { parsed_lists_meta_value.InitialMetaValue(); - s = db_->Put(default_write_options_, handles_[kListsMetaCF], base_meta_key.Encode(), meta_value); + s = db_->Put(default_write_options_, handles_[kMetaCF], base_meta_key.Encode(), meta_value); } } return s; @@ -991,9 +1081,14 @@ Status Redis::ListsDel(const Slice& key) { ScopeRecordLock l(lock_mgr_, key); BaseMetaKey base_meta_key(key); - Status s = db_->Get(default_read_options_, handles_[kListsMetaCF], base_meta_key.Encode(), &meta_value); + Status s = db_->Get(default_read_options_, handles_[kMetaCF], base_meta_key.Encode(), &meta_value); if (s.ok()) { + auto type = static_cast(static_cast(meta_value[0])); + if (type != Type::kList) { + return Status::InvalidArgument("WRONGTYPE Operation against a key holding the wrong kind of value"); + } ParsedListsMetaValue parsed_lists_meta_value(&meta_value); + if (parsed_lists_meta_value.IsStale()) { return Status::NotFound("Stale"); } else if (parsed_lists_meta_value.Count() == 0) { @@ -1001,7 +1096,7 @@ Status Redis::ListsDel(const Slice& key) { } else { uint32_t statistic = parsed_lists_meta_value.Count(); parsed_lists_meta_value.InitialMetaValue(); - s = db_->Put(default_write_options_, handles_[kListsMetaCF], base_meta_key.Encode(), meta_value); + s = db_->Put(default_write_options_, handles_[kMetaCF], base_meta_key.Encode(), meta_value); UpdateSpecificKeyStatistics(DataType::kLists, key.ToString(), statistic); } } @@ -1013,9 +1108,14 @@ Status Redis::ListsExpireat(const Slice& key, uint64_t timestamp) { ScopeRecordLock l(lock_mgr_, key); BaseMetaKey base_meta_key(key); - Status s = db_->Get(default_read_options_, handles_[kListsMetaCF], base_meta_key.Encode(), &meta_value); + Status s = db_->Get(default_read_options_, handles_[kMetaCF], base_meta_key.Encode(), &meta_value); if (s.ok()) { + auto type = static_cast(static_cast(meta_value[0])); + if (type != Type::kList) { + return Status::InvalidArgument("WRONGTYPE Operation against a key holding the wrong kind of value"); + } ParsedListsMetaValue parsed_lists_meta_value(&meta_value); + if (parsed_lists_meta_value.IsStale()) { return Status::NotFound("Stale"); } else if (parsed_lists_meta_value.Count() == 0) { @@ -1026,7 +1126,7 @@ Status Redis::ListsExpireat(const Slice& key, uint64_t timestamp) { } else { parsed_lists_meta_value.InitialMetaValue(); } - return db_->Put(default_write_options_, handles_[kListsMetaCF], base_meta_key.Encode(), meta_value); + return db_->Put(default_write_options_, handles_[kMetaCF], base_meta_key.Encode(), meta_value); } } return s; @@ -1036,9 +1136,14 @@ Status Redis::ListsPersist(const Slice& key) { std::string meta_value; ScopeRecordLock l(lock_mgr_, key); BaseMetaKey base_meta_key(key); - Status s = db_->Get(default_read_options_, handles_[kListsMetaCF], base_meta_key.Encode(), &meta_value); + Status s = db_->Get(default_read_options_, handles_[kMetaCF], base_meta_key.Encode(), &meta_value); if (s.ok()) { + auto type = static_cast(static_cast(meta_value[0])); + if (type != Type::kList) { + return Status::InvalidArgument("WRONGTYPE Operation against a key holding the wrong kind of value"); + } ParsedListsMetaValue parsed_lists_meta_value(&meta_value); + if (parsed_lists_meta_value.IsStale()) { return Status::NotFound("Stale"); } else if (parsed_lists_meta_value.Count() == 0) { @@ -1049,7 +1154,7 @@ Status Redis::ListsPersist(const Slice& key) { return Status::NotFound("Not have an associated timeout"); } else { parsed_lists_meta_value.SetEtime(0); - return db_->Put(default_write_options_, handles_[kListsMetaCF], base_meta_key.Encode(), meta_value); + return db_->Put(default_write_options_, handles_[kMetaCF], base_meta_key.Encode(), meta_value); } } } @@ -1060,9 +1165,14 @@ Status Redis::ListsTTL(const Slice& key, uint64_t* timestamp) { std::string meta_value; BaseMetaKey base_meta_key(key); - Status s = db_->Get(default_read_options_, handles_[kListsMetaCF], base_meta_key.Encode(), &meta_value); + Status s = db_->Get(default_read_options_, handles_[kMetaCF], base_meta_key.Encode(), &meta_value); if (s.ok()) { + auto type = static_cast(static_cast(meta_value[0])); + if (type != Type::kList) { + return Status::InvalidArgument("WRONGTYPE Operation against a key holding the wrong kind of value"); + } ParsedListsMetaValue parsed_lists_meta_value(&meta_value); + if (parsed_lists_meta_value.IsStale()) { *timestamp = -2; return Status::NotFound("Stale"); @@ -1093,7 +1203,7 @@ Status Redis::ListsRename(const Slice& key, Redis* new_inst, const Slice& newkey BaseMetaKey base_meta_key(key); BaseMetaKey base_meta_newkey(newkey); - Status s = db_->Get(default_read_options_, handles_[kListsMetaCF], base_meta_key.Encode(), &meta_value); + Status s = db_->Get(default_read_options_, handles_[kMetaCF], base_meta_key.Encode(), &meta_value); if (s.ok()) { ParsedListsMetaValue parsed_lists_meta_value(&meta_value); if (parsed_lists_meta_value.IsStale()) { @@ -1103,12 +1213,12 @@ Status Redis::ListsRename(const Slice& key, Redis* new_inst, const Slice& newkey } // copy a new list with newkey statistic = parsed_lists_meta_value.Count(); - s = new_inst->GetDB()->Put(default_write_options_, handles_[kListsMetaCF], base_meta_newkey.Encode(), meta_value); + s = new_inst->GetDB()->Put(default_write_options_, handles_[kMetaCF], base_meta_newkey.Encode(), meta_value); new_inst->UpdateSpecificKeyStatistics(DataType::kLists, newkey.ToString(), statistic); // ListsDel key parsed_lists_meta_value.InitialMetaValue(); - s = db_->Put(default_write_options_, handles_[kListsMetaCF], base_meta_key.Encode(), meta_value); + s = db_->Put(default_write_options_, handles_[kMetaCF], base_meta_key.Encode(), meta_value); UpdateSpecificKeyStatistics(DataType::kLists, key.ToString(), statistic); } return s; @@ -1122,7 +1232,7 @@ Status Redis::ListsRenamenx(const Slice& key, Redis* new_inst, const Slice& newk BaseMetaKey base_meta_key(key); BaseMetaKey base_meta_newkey(newkey); - Status s = db_->Get(default_read_options_, handles_[kListsMetaCF], base_meta_key.Encode(), &meta_value); + Status s = db_->Get(default_read_options_, handles_[kMetaCF], base_meta_key.Encode(), &meta_value); if (s.ok()) { ParsedListsMetaValue parsed_lists_meta_value(&meta_value); if (parsed_lists_meta_value.IsStale()) { @@ -1132,7 +1242,7 @@ Status Redis::ListsRenamenx(const Slice& key, Redis* new_inst, const Slice& newk } // check if newkey exists. std::string new_meta_value; - s = new_inst->GetDB()->Get(default_read_options_, handles_[kListsMetaCF], base_meta_newkey.Encode(), + s = new_inst->GetDB()->Get(default_read_options_, handles_[kMetaCF], base_meta_newkey.Encode(), &new_meta_value); if (s.ok()) { ParsedSetsMetaValue parsed_lists_new_meta_value(&new_meta_value); @@ -1143,12 +1253,12 @@ Status Redis::ListsRenamenx(const Slice& key, Redis* new_inst, const Slice& newk // copy a new list with newkey statistic = parsed_lists_meta_value.Count(); - s = new_inst->GetDB()->Put(default_write_options_, handles_[kListsMetaCF], base_meta_newkey.Encode(), meta_value); + s = new_inst->GetDB()->Put(default_write_options_, handles_[kMetaCF], base_meta_newkey.Encode(), meta_value); new_inst->UpdateSpecificKeyStatistics(DataType::kLists, newkey.ToString(), statistic); // ListsDel key parsed_lists_meta_value.InitialMetaValue(); - s = db_->Put(default_write_options_, handles_[kListsMetaCF], base_meta_key.Encode(), meta_value); + s = db_->Put(default_write_options_, handles_[kMetaCF], base_meta_key.Encode(), meta_value); UpdateSpecificKeyStatistics(DataType::kLists, key.ToString(), statistic); } return s; @@ -1163,9 +1273,12 @@ void Redis::ScanLists() { auto current_time = static_cast(time(nullptr)); INFO("***************rocksdb instance: {} List Meta Data***************", index_); - auto meta_iter = db_->NewIterator(iterator_options, handles_[kListsMetaCF]); + auto meta_iter = db_->NewIterator(iterator_options, handles_[kMetaCF]); for (meta_iter->SeekToFirst(); meta_iter->Valid(); meta_iter->Next()) { ParsedListsMetaValue parsed_lists_meta_value(meta_iter->value()); + if (!parsed_lists_meta_value.IsSameType(Type::kList)) { + continue; + } ParsedBaseMetaKey parsed_meta_key(meta_iter->value()); int32_t survival_time = 0; if (parsed_lists_meta_value.Etime() != 0) { diff --git a/src/storage/src/redis_sets.cc b/src/storage/src/redis_sets.cc index 2ec883af8..567561590 100644 --- a/src/storage/src/redis_sets.cc +++ b/src/storage/src/redis_sets.cc @@ -38,9 +38,12 @@ rocksdb::Status Redis::ScanSetsKeyNum(KeyInfo* key_info) { int64_t curtime; rocksdb::Env::Default()->GetCurrentTime(&curtime); - rocksdb::Iterator* iter = db_->NewIterator(iterator_options, handles_[kSetsMetaCF]); + rocksdb::Iterator* iter = db_->NewIterator(iterator_options, handles_[kMetaCF]); for (iter->SeekToFirst(); iter->Valid(); iter->Next()) { ParsedSetsMetaValue parsed_sets_meta_value(iter->value()); + if (!parsed_sets_meta_value.IsSameType(Type::kSet)) { + continue; + } if (parsed_sets_meta_value.IsStale() || parsed_sets_meta_value.Count() == 0) { invaild_keys++; } else { @@ -72,17 +75,20 @@ rocksdb::Status Redis::SetsPKPatternMatchDel(const std::string& pattern, int32_t int32_t total_delete = 0; rocksdb::Status s; rocksdb::WriteBatch batch; - rocksdb::Iterator* iter = db_->NewIterator(iterator_options, handles_[kSetsMetaCF]); + rocksdb::Iterator* iter = db_->NewIterator(iterator_options, handles_[kMetaCF]); iter->SeekToFirst(); while (iter->Valid()) { ParsedBaseMetaKey parsed_meta_key(iter->key()); meta_value = iter->value().ToString(); ParsedSetsMetaValue parsed_sets_meta_value(&meta_value); + if (!parsed_sets_meta_value.IsSameType(Type::kSet)) { + continue; + } if (!parsed_sets_meta_value.IsStale() && (parsed_sets_meta_value.Count() != 0) && (StringMatch(pattern.data(), pattern.size(), parsed_meta_key.Key().data(), parsed_meta_key.Key().size(), 0) != 0)) { parsed_sets_meta_value.InitialMetaValue(); - batch.Put(handles_[kSetsMetaCF], iter->key(), meta_value); + batch.Put(handles_[kMetaCF], iter->key(), meta_value); } if (static_cast(batch.Count()) >= BATCH_DELETE_LIMIT) { s = db_->Write(default_write_options_, &batch); @@ -124,8 +130,12 @@ rocksdb::Status Redis::SAdd(const Slice& key, const std::vector& me std::string meta_value; BaseMetaKey base_meta_key(key); - rocksdb::Status s = db_->Get(default_read_options_, handles_[kSetsMetaCF], base_meta_key.Encode(), &meta_value); + rocksdb::Status s = db_->Get(default_read_options_, handles_[kMetaCF], base_meta_key.Encode(), &meta_value); if (s.ok()) { + auto type = static_cast(static_cast(meta_value[0])); + if (type != Type::kSet) { + return Status::InvalidArgument("WRONGTYPE Operation against a key holding the wrong kind of value"); + } ParsedSetsMetaValue parsed_sets_meta_value(&meta_value); if (parsed_sets_meta_value.IsStale() || parsed_sets_meta_value.Count() == 0) { version = parsed_sets_meta_value.InitialMetaValue(); @@ -133,7 +143,7 @@ rocksdb::Status Redis::SAdd(const Slice& key, const std::vector& me return Status::InvalidArgument("set size overflow"); } parsed_sets_meta_value.SetCount(static_cast(filtered_members.size())); - batch->Put(kSetsMetaCF, base_meta_key.Encode(), meta_value); + batch->Put(kMetaCF, base_meta_key.Encode(), meta_value); for (const auto& member : filtered_members) { SetsMemberKey sets_member_key(key, version, member); BaseDataValue iter_value(Slice{}); @@ -164,15 +174,15 @@ rocksdb::Status Redis::SAdd(const Slice& key, const std::vector& me return Status::InvalidArgument("set size overflow"); } parsed_sets_meta_value.ModifyCount(cnt); - batch->Put(kSetsMetaCF, base_meta_key.Encode(), meta_value); + batch->Put(kMetaCF, base_meta_key.Encode(), meta_value); } } } else if (s.IsNotFound()) { char str[4]; EncodeFixed32(str, filtered_members.size()); - SetsMetaValue sets_meta_value(Slice(str, sizeof(int32_t))); + SetsMetaValue sets_meta_value(Type::kSet, Slice(str, 4)); version = sets_meta_value.UpdateVersion(); - batch->Put(kSetsMetaCF, base_meta_key.Encode(), sets_meta_value.Encode()); + batch->Put(kMetaCF, base_meta_key.Encode(), sets_meta_value.Encode()); for (const auto& member : filtered_members) { SetsMemberKey sets_member_key(key, version, member); BaseDataValue i_val(Slice{}); @@ -190,8 +200,12 @@ rocksdb::Status Redis::SCard(const Slice& key, int32_t* ret) { std::string meta_value; BaseMetaKey base_meta_key(key); - rocksdb::Status s = db_->Get(default_read_options_, handles_[kSetsMetaCF], base_meta_key.Encode(), &meta_value); + rocksdb::Status s = db_->Get(default_read_options_, handles_[kMetaCF], base_meta_key.Encode(), &meta_value); if (s.ok()) { + auto type = static_cast(static_cast(meta_value[0])); + if (type != Type::kSet) { + return Status::InvalidArgument("WRONGTYPE Operation against a key holding the wrong kind of value"); + } ParsedSetsMetaValue parsed_sets_meta_value(&meta_value); if (parsed_sets_meta_value.IsStale()) { return rocksdb::Status::NotFound("Stale"); @@ -222,9 +236,12 @@ rocksdb::Status Redis::SDiff(const std::vector& keys, std::vectorGet(read_options, handles_[kSetsMetaCF], base_meta_key.Encode(), &meta_value); + s = db_->Get(read_options, handles_[kMetaCF], base_meta_key.Encode(), &meta_value); if (s.ok()) { ParsedSetsMetaValue parsed_sets_meta_value(&meta_value); + if (!parsed_sets_meta_value.IsSameType(Type::kSet)) { + return Status::InvalidArgument("WRONGTYPE Operation against a key holding the wrong kind of value"); + } if (!parsed_sets_meta_value.IsStale() && parsed_sets_meta_value.Count() != 0) { vaild_sets.push_back({keys[idx], parsed_sets_meta_value.Version()}); } @@ -234,8 +251,12 @@ rocksdb::Status Redis::SDiff(const std::vector& keys, std::vectorGet(read_options, handles_[kSetsMetaCF], base_meta_key0.Encode(), &meta_value); + s = db_->Get(read_options, handles_[kMetaCF], base_meta_key0.Encode(), &meta_value); if (s.ok()) { + auto type = static_cast(static_cast(meta_value[0])); + if (type != Type::kSet) { + return Status::InvalidArgument("WRONGTYPE Operation against a key holding the wrong kind of value"); + } ParsedSetsMetaValue parsed_sets_meta_value(&meta_value); if (!parsed_sets_meta_value.IsStale() && parsed_sets_meta_value.Count() != 0) { bool found; @@ -294,9 +315,12 @@ rocksdb::Status Redis::SDiffstore(const Slice& destination, const std::vectorGet(read_options, handles_[kSetsMetaCF], base_meta_key.Encode(), &meta_value); + s = db_->Get(read_options, handles_[kMetaCF], base_meta_key.Encode(), &meta_value); if (s.ok()) { ParsedSetsMetaValue parsed_sets_meta_value(&meta_value); + if (!parsed_sets_meta_value.IsSameType(Type::kSet)) { + return Status::InvalidArgument("WRONGTYPE Operation against a key holding the wrong kind of value"); + } if (!parsed_sets_meta_value.IsStale() && parsed_sets_meta_value.Count() != 0) { vaild_sets.push_back({keys[idx], parsed_sets_meta_value.Version()}); } @@ -307,8 +331,12 @@ rocksdb::Status Redis::SDiffstore(const Slice& destination, const std::vector members; BaseMetaKey base_meta_key0(keys[0]); - s = db_->Get(read_options, handles_[kSetsMetaCF], base_meta_key0.Encode(), &meta_value); + s = db_->Get(read_options, handles_[kMetaCF], base_meta_key0.Encode(), &meta_value); if (s.ok()) { + auto type = static_cast(static_cast(meta_value[0])); + if (type != Type::kSet) { + return Status::InvalidArgument("WRONGTYPE Operation against a key holding the wrong kind of value"); + } ParsedSetsMetaValue parsed_sets_meta_value(&meta_value); if (!parsed_sets_meta_value.IsStale() && parsed_sets_meta_value.Count() != 0) { bool found; @@ -346,8 +374,12 @@ rocksdb::Status Redis::SDiffstore(const Slice& destination, const std::vectorGet(read_options, handles_[kSetsMetaCF], base_destination.Encode(), &meta_value); + s = db_->Get(read_options, handles_[kMetaCF], base_destination.Encode(), &meta_value); if (s.ok()) { + auto type = static_cast(static_cast(meta_value[0])); + if (type != Type::kSet) { + return Status::InvalidArgument("WRONGTYPE Operation against a key holding the wrong kind of value"); + } ParsedSetsMetaValue parsed_sets_meta_value(&meta_value); statistic = parsed_sets_meta_value.Count(); version = parsed_sets_meta_value.InitialMetaValue(); @@ -355,13 +387,13 @@ rocksdb::Status Redis::SDiffstore(const Slice& destination, const std::vector(members.size())); - batch->Put(kSetsMetaCF, base_destination.Encode(), meta_value); + batch->Put(kMetaCF, base_destination.Encode(), meta_value); } else if (s.IsNotFound()) { char str[4]; EncodeFixed32(str, members.size()); - SetsMetaValue sets_meta_value(Slice(str, sizeof(int32_t))); + SetsMetaValue sets_meta_value(Type::kSet, Slice(str, 4)); version = sets_meta_value.UpdateVersion(); - batch->Put(kSetsMetaCF, base_destination.Encode(), sets_meta_value.Encode()); + batch->Put(kMetaCF, base_destination.Encode(), sets_meta_value.Encode()); } else { return s; } @@ -394,9 +426,12 @@ rocksdb::Status Redis::SInter(const std::vector& keys, std::vector< for (uint32_t idx = 1; idx < keys.size(); ++idx) { BaseMetaKey base_meta_key(keys[idx]); - s = db_->Get(read_options, handles_[kSetsMetaCF], base_meta_key.Encode(), &meta_value); + s = db_->Get(read_options, handles_[kMetaCF], base_meta_key.Encode(), &meta_value); if (s.ok()) { ParsedSetsMetaValue parsed_sets_meta_value(&meta_value); + if (!parsed_sets_meta_value.IsSameType(Type::kSet)) { + return Status::InvalidArgument("WRONGTYPE Operation against a key holding the wrong kind of value"); + } if (parsed_sets_meta_value.IsStale() || parsed_sets_meta_value.Count() == 0) { return rocksdb::Status::OK(); } else { @@ -410,8 +445,12 @@ rocksdb::Status Redis::SInter(const std::vector& keys, std::vector< } BaseMetaKey base_meta_key0(keys[0]); - s = db_->Get(read_options, handles_[kSetsMetaCF], base_meta_key0.Encode(), &meta_value); + s = db_->Get(read_options, handles_[kMetaCF], base_meta_key0.Encode(), &meta_value); if (s.ok()) { + auto type = static_cast(static_cast(meta_value[0])); + if (type != Type::kSet) { + return Status::InvalidArgument("WRONGTYPE Operation against a key holding the wrong kind of value"); + } ParsedSetsMetaValue parsed_sets_meta_value(&meta_value); if (parsed_sets_meta_value.IsStale() || parsed_sets_meta_value.Count() == 0) { return rocksdb::Status::OK(); @@ -476,9 +515,12 @@ rocksdb::Status Redis::SInterstore(const Slice& destination, const std::vectorGet(read_options, handles_[kSetsMetaCF], base_meta_key.Encode(), &meta_value); + s = db_->Get(read_options, handles_[kMetaCF], base_meta_key.Encode(), &meta_value); if (s.ok()) { ParsedSetsMetaValue parsed_sets_meta_value(&meta_value); + if (!parsed_sets_meta_value.IsSameType(Type::kSet)) { + return Status::InvalidArgument("WRONGTYPE Operation against a key holding the wrong kind of value"); + } if (parsed_sets_meta_value.IsStale() || parsed_sets_meta_value.Count() == 0) { have_invalid_sets = true; break; @@ -496,9 +538,12 @@ rocksdb::Status Redis::SInterstore(const Slice& destination, const std::vector members; if (!have_invalid_sets) { BaseMetaKey base_meta_key0(keys[0]); - s = db_->Get(read_options, handles_[kSetsMetaCF], base_meta_key0.Encode(), &meta_value); + s = db_->Get(read_options, handles_[kMetaCF], base_meta_key0.Encode(), &meta_value); if (s.ok()) { ParsedSetsMetaValue parsed_sets_meta_value(&meta_value); + if (!parsed_sets_meta_value.IsSameType(Type::kSet)) { + return Status::InvalidArgument("WRONGTYPE Operation against a key holding the wrong kind of value"); + } if (parsed_sets_meta_value.IsStale() || parsed_sets_meta_value.Count() == 0) { have_invalid_sets = true; } else { @@ -541,8 +586,12 @@ rocksdb::Status Redis::SInterstore(const Slice& destination, const std::vectorGet(read_options, handles_[kSetsMetaCF], base_destination.Encode(), &meta_value); + s = db_->Get(read_options, handles_[kMetaCF], base_destination.Encode(), &meta_value); if (s.ok()) { + auto type = static_cast(static_cast(meta_value[0])); + if (type != Type::kSet) { + return Status::InvalidArgument("WRONGTYPE Operation against a key holding the wrong kind of value"); + } ParsedSetsMetaValue parsed_sets_meta_value(&meta_value); statistic = parsed_sets_meta_value.Count(); version = parsed_sets_meta_value.InitialMetaValue(); @@ -550,13 +599,13 @@ rocksdb::Status Redis::SInterstore(const Slice& destination, const std::vector(members.size())); - batch->Put(kSetsMetaCF, base_destination.Encode(), meta_value); + batch->Put(kMetaCF, base_destination.Encode(), meta_value); } else if (s.IsNotFound()) { - char str[4]; + char str[5]; EncodeFixed32(str, members.size()); - SetsMetaValue sets_meta_value(Slice(str, sizeof(int32_t))); + SetsMetaValue sets_meta_value(Type::kSet, Slice(str, 4)); version = sets_meta_value.UpdateVersion(); - batch->Put(kSetsMetaCF, base_destination.Encode(), sets_meta_value.Encode()); + batch->Put(kMetaCF, base_destination.Encode(), sets_meta_value.Encode()); } else { return s; } @@ -583,8 +632,12 @@ rocksdb::Status Redis::SIsmember(const Slice& key, const Slice& member, int32_t* read_options.snapshot = snapshot; BaseMetaKey base_meta_key(key); - rocksdb::Status s = db_->Get(read_options, handles_[kSetsMetaCF], base_meta_key.Encode(), &meta_value); + rocksdb::Status s = db_->Get(read_options, handles_[kMetaCF], base_meta_key.Encode(), &meta_value); if (s.ok()) { + auto type = static_cast(static_cast(meta_value[0])); + if (type != Type::kSet) { + return Status::InvalidArgument("WRONGTYPE Operation against a key holding the wrong kind of value"); + } ParsedSetsMetaValue parsed_sets_meta_value(&meta_value); if (parsed_sets_meta_value.IsStale()) { return rocksdb::Status::NotFound("Stale"); @@ -613,8 +666,12 @@ rocksdb::Status Redis::SMembers(const Slice& key, std::vector* memb read_options.snapshot = snapshot; BaseMetaKey base_meta_key(key); - rocksdb::Status s = db_->Get(read_options, handles_[kSetsMetaCF], base_meta_key.Encode(), &meta_value); + rocksdb::Status s = db_->Get(read_options, handles_[kMetaCF], base_meta_key.Encode(), &meta_value); if (s.ok()) { + auto type = static_cast(static_cast(meta_value[0])); + if (type != Type::kSet) { + return Status::InvalidArgument("WRONGTYPE Operation against a key holding the wrong kind of value"); + } ParsedSetsMetaValue parsed_sets_meta_value(&meta_value); if (parsed_sets_meta_value.IsStale()) { return rocksdb::Status::NotFound("Stale"); @@ -645,8 +702,12 @@ Status Redis::SMembersWithTTL(const Slice& key, std::vector* member ScopeSnapshot ss(db_, &snapshot); read_options.snapshot = snapshot; BaseMetaKey base_meta_key(key); - rocksdb::Status s = db_->Get(read_options, handles_[kSetsMetaCF], base_meta_key.Encode(), &meta_value); + rocksdb::Status s = db_->Get(read_options, handles_[kMetaCF], base_meta_key.Encode(), &meta_value); if (s.ok()) { + auto type = static_cast(static_cast(meta_value[0])); + if (type != Type::kSet) { + return Status::InvalidArgument("WRONGTYPE Operation against a key holding the wrong kind of value"); + } ParsedSetsMetaValue parsed_sets_meta_value(&meta_value); if (parsed_sets_meta_value.Count() == 0) { return Status::NotFound(); @@ -695,8 +756,12 @@ rocksdb::Status Redis::SMove(const Slice& source, const Slice& destination, cons } BaseMetaKey base_source(source); - rocksdb::Status s = db_->Get(default_read_options_, handles_[kSetsMetaCF], base_source.Encode(), &meta_value); + rocksdb::Status s = db_->Get(default_read_options_, handles_[kMetaCF], base_source.Encode(), &meta_value); if (s.ok()) { + auto type = static_cast(static_cast(meta_value[0])); + if (type != Type::kSet) { + return Status::InvalidArgument("WRONGTYPE Operation against a key holding the wrong kind of value"); + } ParsedSetsMetaValue parsed_sets_meta_value(&meta_value); if (parsed_sets_meta_value.IsStale()) { return rocksdb::Status::NotFound("Stale"); @@ -713,7 +778,7 @@ rocksdb::Status Redis::SMove(const Slice& source, const Slice& destination, cons return Status::InvalidArgument("set size overflow"); } parsed_sets_meta_value.ModifyCount(-1); - batch->Put(kSetsMetaCF, base_source.Encode(), meta_value); + batch->Put(kMetaCF, base_source.Encode(), meta_value); batch->Delete(kSetsDataCF, sets_member_key.Encode()); statistic++; } else if (s.IsNotFound()) { @@ -731,13 +796,17 @@ rocksdb::Status Redis::SMove(const Slice& source, const Slice& destination, cons } BaseMetaKey base_destination(destination); - s = db_->Get(default_read_options_, handles_[kSetsMetaCF], base_destination.Encode(), &meta_value); + s = db_->Get(default_read_options_, handles_[kMetaCF], base_destination.Encode(), &meta_value); if (s.ok()) { + auto type = static_cast(static_cast(meta_value[0])); + if (type != Type::kSet) { + return Status::InvalidArgument("WRONGTYPE Operation against a key holding the wrong kind of value"); + } ParsedSetsMetaValue parsed_sets_meta_value(&meta_value); if (parsed_sets_meta_value.IsStale() || parsed_sets_meta_value.Count() == 0) { version = parsed_sets_meta_value.InitialMetaValue(); parsed_sets_meta_value.SetCount(1); - batch->Put(kSetsMetaCF, base_destination.Encode(), meta_value); + batch->Put(kMetaCF, base_destination.Encode(), meta_value); SetsMemberKey sets_member_key(destination, version, member); BaseDataValue i_val(Slice{}); batch->Put(kSetsDataCF, sets_member_key.Encode(), i_val.Encode()); @@ -752,18 +821,18 @@ rocksdb::Status Redis::SMove(const Slice& source, const Slice& destination, cons } parsed_sets_meta_value.ModifyCount(1); BaseDataValue iter_value(Slice{}); - batch->Put(kSetsMetaCF, base_destination.Encode(), meta_value); + batch->Put(kMetaCF, base_destination.Encode(), meta_value); batch->Put(kSetsDataCF, sets_member_key.Encode(), iter_value.Encode()); } else if (!s.ok()) { return s; } } } else if (s.IsNotFound()) { - char str[4]; + char str[5]; EncodeFixed32(str, 1); - SetsMetaValue sets_meta_value(Slice(str, sizeof(int32_t))); + SetsMetaValue sets_meta_value(Type::kSet, Slice(str, 4)); version = sets_meta_value.UpdateVersion(); - batch->Put(kSetsMetaCF, base_destination.Encode(), sets_meta_value.Encode()); + batch->Put(kMetaCF, base_destination.Encode(), sets_meta_value.Encode()); SetsMemberKey sets_member_key(destination, version, member); BaseDataValue iter_value(Slice{}); batch->Put(kSetsDataCF, sets_member_key.Encode(), iter_value.Encode()); @@ -785,8 +854,12 @@ rocksdb::Status Redis::SPop(const Slice& key, std::vector* members, uint64_t start_us = pstd::NowMicros(); BaseMetaKey base_meta_key(key); - Status s = db_->Get(default_read_options_, handles_[kSetsMetaCF], base_meta_key.Encode(), &meta_value); + Status s = db_->Get(default_read_options_, handles_[kMetaCF], base_meta_key.Encode(), &meta_value); if (s.ok()) { + auto type = static_cast(static_cast(meta_value[0])); + if (type != Type::kSet) { + return Status::InvalidArgument("WRONGTYPE Operation against a key holding the wrong kind of value"); + } ParsedSetsMetaValue parsed_sets_meta_value(&meta_value); if (parsed_sets_meta_value.IsStale()) { return Status::NotFound("Stale"); @@ -809,7 +882,7 @@ rocksdb::Status Redis::SPop(const Slice& key, std::vector* members, // parsed_sets_meta_value.ModifyCount(-cnt); // batch.Put(handles_[kSetsMetaCF], key, meta_value); - batch->Delete(kSetsMetaCF, base_meta_key.Encode()); + batch->Delete(kMetaCF, base_meta_key.Encode()); delete iter; } else { @@ -849,7 +922,7 @@ rocksdb::Status Redis::SPop(const Slice& key, std::vector* members, return Status::InvalidArgument("set size overflow"); } parsed_sets_meta_value.ModifyCount(static_cast(-cnt)); - batch->Put(kSetsMetaCF, base_meta_key.Encode(), meta_value); + batch->Put(kMetaCF, base_meta_key.Encode(), meta_value); delete iter; } } @@ -885,8 +958,12 @@ rocksdb::Status Redis::SRandmember(const Slice& key, int32_t count, std::vector< std::unordered_set unique; BaseMetaKey base_meta_key(key); - rocksdb::Status s = db_->Get(default_read_options_, handles_[kSetsMetaCF], base_meta_key.Encode(), &meta_value); + rocksdb::Status s = db_->Get(default_read_options_, handles_[kMetaCF], base_meta_key.Encode(), &meta_value); if (s.ok()) { + auto type = static_cast(static_cast(meta_value[0])); + if (type != Type::kSet) { + return Status::InvalidArgument("WRONGTYPE Operation against a key holding the wrong kind of value"); + } ParsedSetsMetaValue parsed_sets_meta_value(&meta_value); if (parsed_sets_meta_value.IsStale()) { return rocksdb::Status::NotFound("Stale"); @@ -949,8 +1026,12 @@ rocksdb::Status Redis::SRem(const Slice& key, const std::vector& me std::string meta_value; BaseMetaKey base_meta_key(key); - rocksdb::Status s = db_->Get(default_read_options_, handles_[kSetsMetaCF], base_meta_key.Encode(), &meta_value); + rocksdb::Status s = db_->Get(default_read_options_, handles_[kMetaCF], base_meta_key.Encode(), &meta_value); if (s.ok()) { + auto type = static_cast(static_cast(meta_value[0])); + if (type != Type::kSet) { + return Status::InvalidArgument("WRONGTYPE Operation against a key holding the wrong kind of value"); + } ParsedSetsMetaValue parsed_sets_meta_value(&meta_value); if (parsed_sets_meta_value.IsStale()) { return rocksdb::Status::NotFound("stale"); @@ -977,7 +1058,7 @@ rocksdb::Status Redis::SRem(const Slice& key, const std::vector& me return Status::InvalidArgument("set size overflow"); } parsed_sets_meta_value.ModifyCount(-cnt); - batch->Put(kSetsMetaCF, base_meta_key.Encode(), meta_value); + batch->Put(kMetaCF, base_meta_key.Encode(), meta_value); } } else if (s.IsNotFound()) { *ret = 0; @@ -1006,9 +1087,12 @@ rocksdb::Status Redis::SUnion(const std::vector& keys, std::vector< for (const auto& key : keys) { BaseMetaKey base_meta_key(key); - s = db_->Get(read_options, handles_[kSetsMetaCF], base_meta_key.Encode(), &meta_value); + s = db_->Get(read_options, handles_[kMetaCF], base_meta_key.Encode(), &meta_value); if (s.ok()) { ParsedSetsMetaValue parsed_sets_meta_value(&meta_value); + if (!parsed_sets_meta_value.IsSameType(Type::kSet)) { + return Status::InvalidArgument("WRONGTYPE Operation against a key holding the wrong kind of value"); + } if (!parsed_sets_meta_value.IsStale() && parsed_sets_meta_value.Count() != 0) { vaild_sets.push_back({key, parsed_sets_meta_value.Version()}); } @@ -1057,9 +1141,12 @@ rocksdb::Status Redis::SUnionstore(const Slice& destination, const std::vectorGet(read_options, handles_[kSetsMetaCF], base_meta_key.Encode(), &meta_value); + s = db_->Get(read_options, handles_[kMetaCF], base_meta_key.Encode(), &meta_value); if (s.ok()) { ParsedSetsMetaValue parsed_sets_meta_value(&meta_value); + if (!parsed_sets_meta_value.IsSameType(Type::kSet)) { + return Status::InvalidArgument("WRONGTYPE Operation against a key holding the wrong kind of value"); + } if (!parsed_sets_meta_value.IsStale() && parsed_sets_meta_value.Count() != 0) { vaild_sets.push_back({key, parsed_sets_meta_value.Version()}); } @@ -1089,8 +1176,12 @@ rocksdb::Status Redis::SUnionstore(const Slice& destination, const std::vectorGet(read_options, handles_[kSetsMetaCF], base_destination.Encode(), &meta_value); + s = db_->Get(read_options, handles_[kMetaCF], base_destination.Encode(), &meta_value); if (s.ok()) { + auto type = static_cast(static_cast(meta_value[0])); + if (type != Type::kSet) { + return Status::InvalidArgument("WRONGTYPE Operation against a key holding the wrong kind of value"); + } ParsedSetsMetaValue parsed_sets_meta_value(&meta_value); statistic = parsed_sets_meta_value.Count(); version = parsed_sets_meta_value.InitialMetaValue(); @@ -1098,13 +1189,13 @@ rocksdb::Status Redis::SUnionstore(const Slice& destination, const std::vector(members.size())); - batch->Put(kSetsMetaCF, destination, meta_value); + batch->Put(kMetaCF, destination, meta_value); } else if (s.IsNotFound()) { - char str[4]; + char str[5]; EncodeFixed32(str, members.size()); - SetsMetaValue sets_meta_value(Slice(str, sizeof(int32_t))); + SetsMetaValue sets_meta_value(Type::kSet, Slice(str, 4)); version = sets_meta_value.UpdateVersion(); - batch->Put(kSetsMetaCF, base_destination.Encode(), sets_meta_value.Encode()); + batch->Put(kMetaCF, base_destination.Encode(), sets_meta_value.Encode()); } else { return s; } @@ -1139,8 +1230,12 @@ rocksdb::Status Redis::SScan(const Slice& key, int64_t cursor, const std::string read_options.snapshot = snapshot; BaseMetaKey base_meta_key(key); - rocksdb::Status s = db_->Get(read_options, handles_[kSetsMetaCF], base_meta_key.Encode(), &meta_value); + rocksdb::Status s = db_->Get(read_options, handles_[kMetaCF], base_meta_key.Encode(), &meta_value); if (s.ok()) { + auto type = static_cast(static_cast(meta_value[0])); + if (type != Type::kSet) { + return Status::InvalidArgument("WRONGTYPE Operation against a key holding the wrong kind of value"); + } ParsedSetsMetaValue parsed_sets_meta_value(&meta_value); if (parsed_sets_meta_value.IsStale() || parsed_sets_meta_value.Count() == 0) { *next_cursor = 0; @@ -1197,8 +1292,12 @@ rocksdb::Status Redis::SetsExpire(const Slice& key, uint64_t ttl) { ScopeRecordLock l(lock_mgr_, key); BaseMetaKey base_meta_key(key); - rocksdb::Status s = db_->Get(default_read_options_, handles_[kSetsMetaCF], base_meta_key.Encode(), &meta_value); + rocksdb::Status s = db_->Get(default_read_options_, handles_[kMetaCF], base_meta_key.Encode(), &meta_value); if (s.ok()) { + auto type = static_cast(static_cast(meta_value[0])); + if (type != Type::kSet) { + return Status::InvalidArgument("WRONGTYPE Operation against a key holding the wrong kind of value"); + } ParsedSetsMetaValue parsed_sets_meta_value(&meta_value); if (parsed_sets_meta_value.IsStale()) { return rocksdb::Status::NotFound("Stale"); @@ -1208,10 +1307,10 @@ rocksdb::Status Redis::SetsExpire(const Slice& key, uint64_t ttl) { if (ttl > 0) { parsed_sets_meta_value.SetRelativeTimestamp(ttl); - s = db_->Put(default_write_options_, handles_[kSetsMetaCF], base_meta_key.Encode(), meta_value); + s = db_->Put(default_write_options_, handles_[kMetaCF], base_meta_key.Encode(), meta_value); } else { parsed_sets_meta_value.InitialMetaValue(); - s = db_->Put(default_write_options_, handles_[kSetsMetaCF], base_meta_key.Encode(), meta_value); + s = db_->Put(default_write_options_, handles_[kMetaCF], base_meta_key.Encode(), meta_value); } } return s; @@ -1222,8 +1321,12 @@ rocksdb::Status Redis::SetsDel(const Slice& key) { ScopeRecordLock l(lock_mgr_, key); BaseMetaKey base_meta_key(key); - rocksdb::Status s = db_->Get(default_read_options_, handles_[kSetsMetaCF], base_meta_key.Encode(), &meta_value); + rocksdb::Status s = db_->Get(default_read_options_, handles_[kMetaCF], base_meta_key.Encode(), &meta_value); if (s.ok()) { + auto type = static_cast(static_cast(meta_value[0])); + if (type != Type::kSet) { + return Status::InvalidArgument("WRONGTYPE Operation against a key holding the wrong kind of value"); + } ParsedSetsMetaValue parsed_sets_meta_value(&meta_value); if (parsed_sets_meta_value.IsStale()) { return rocksdb::Status::NotFound("Stale"); @@ -1232,7 +1335,7 @@ rocksdb::Status Redis::SetsDel(const Slice& key) { } else { uint32_t statistic = parsed_sets_meta_value.Count(); parsed_sets_meta_value.InitialMetaValue(); - s = db_->Put(default_write_options_, handles_[kSetsMetaCF], base_meta_key.Encode(), meta_value); + s = db_->Put(default_write_options_, handles_[kMetaCF], base_meta_key.Encode(), meta_value); UpdateSpecificKeyStatistics(DataType::kSets, key.ToString(), statistic); } } @@ -1244,8 +1347,12 @@ rocksdb::Status Redis::SetsExpireat(const Slice& key, uint64_t timestamp) { ScopeRecordLock l(lock_mgr_, key); BaseMetaKey base_meta_key(key); - rocksdb::Status s = db_->Get(default_read_options_, handles_[kSetsMetaCF], base_meta_key.Encode(), &meta_value); + rocksdb::Status s = db_->Get(default_read_options_, handles_[kMetaCF], base_meta_key.Encode(), &meta_value); if (s.ok()) { + auto type = static_cast(static_cast(meta_value[0])); + if (type != Type::kSet) { + return Status::InvalidArgument("WRONGTYPE Operation against a key holding the wrong kind of value"); + } ParsedSetsMetaValue parsed_sets_meta_value(&meta_value); if (parsed_sets_meta_value.IsStale()) { return rocksdb::Status::NotFound("Stale"); @@ -1257,7 +1364,7 @@ rocksdb::Status Redis::SetsExpireat(const Slice& key, uint64_t timestamp) { } else { parsed_sets_meta_value.InitialMetaValue(); } - return db_->Put(default_write_options_, handles_[kSetsMetaCF], base_meta_key.Encode(), meta_value); + return db_->Put(default_write_options_, handles_[kMetaCF], base_meta_key.Encode(), meta_value); } } return s; @@ -1268,8 +1375,12 @@ rocksdb::Status Redis::SetsPersist(const Slice& key) { ScopeRecordLock l(lock_mgr_, key); BaseMetaKey base_meta_key(key); - rocksdb::Status s = db_->Get(default_read_options_, handles_[kSetsMetaCF], base_meta_key.Encode(), &meta_value); + rocksdb::Status s = db_->Get(default_read_options_, handles_[kMetaCF], base_meta_key.Encode(), &meta_value); if (s.ok()) { + auto type = static_cast(static_cast(meta_value[0])); + if (type != Type::kSet) { + return Status::InvalidArgument("WRONGTYPE Operation against a key holding the wrong kind of value"); + } ParsedSetsMetaValue parsed_sets_meta_value(&meta_value); if (parsed_sets_meta_value.IsStale()) { return rocksdb::Status::NotFound("Stale"); @@ -1281,7 +1392,7 @@ rocksdb::Status Redis::SetsPersist(const Slice& key) { return rocksdb::Status::NotFound("Not have an associated timeout"); } else { parsed_sets_meta_value.SetEtime(0); - return db_->Put(default_write_options_, handles_[kSetsMetaCF], base_meta_key.Encode(), meta_value); + return db_->Put(default_write_options_, handles_[kMetaCF], base_meta_key.Encode(), meta_value); } } } @@ -1292,9 +1403,12 @@ rocksdb::Status Redis::SetsTTL(const Slice& key, uint64_t* timestamp) { std::string meta_value; BaseMetaKey base_meta_key(key); - rocksdb::Status s = db_->Get(default_read_options_, handles_[kSetsMetaCF], base_meta_key.Encode(), &meta_value); + rocksdb::Status s = db_->Get(default_read_options_, handles_[kMetaCF], base_meta_key.Encode(), &meta_value); if (s.ok()) { ParsedSetsMetaValue parsed_setes_meta_value(&meta_value); + if (!parsed_setes_meta_value.IsSameType(Type::kSet)) { + return Status::InvalidArgument("WRONGTYPE Operation against a key holding the wrong kind of value"); + } if (parsed_setes_meta_value.IsStale()) { *timestamp = -2; return rocksdb::Status::NotFound("Stale"); @@ -1325,7 +1439,7 @@ Status Redis::SetsRename(const Slice& key, Redis* new_inst, const Slice& newkey) BaseMetaKey base_meta_key(key); BaseMetaKey base_meta_newkey(newkey); - rocksdb::Status s = db_->Get(default_read_options_, handles_[kSetsMetaCF], base_meta_key.Encode(), &meta_value); + rocksdb::Status s = db_->Get(default_read_options_, handles_[kMetaCF], base_meta_key.Encode(), &meta_value); if (s.ok()) { ParsedSetsMetaValue parsed_sets_meta_value(&meta_value); if (parsed_sets_meta_value.IsStale()) { @@ -1335,12 +1449,12 @@ Status Redis::SetsRename(const Slice& key, Redis* new_inst, const Slice& newkey) } // copy a new set with newkey statistic = parsed_sets_meta_value.Count(); - s = new_inst->GetDB()->Put(default_write_options_, handles_[kSetsMetaCF], base_meta_newkey.Encode(), meta_value); + s = new_inst->GetDB()->Put(default_write_options_, handles_[kMetaCF], base_meta_newkey.Encode(), meta_value); new_inst->UpdateSpecificKeyStatistics(DataType::kSets, newkey.ToString(), statistic); // SetsDel key parsed_sets_meta_value.InitialMetaValue(); - s = db_->Put(default_write_options_, handles_[kSetsMetaCF], base_meta_key.Encode(), meta_value); + s = db_->Put(default_write_options_, handles_[kMetaCF], base_meta_key.Encode(), meta_value); UpdateSpecificKeyStatistics(DataType::kSets, key.ToString(), statistic); } return s; @@ -1354,7 +1468,7 @@ Status Redis::SetsRenamenx(const Slice& key, Redis* new_inst, const Slice& newke BaseMetaKey base_meta_key(key); BaseMetaKey base_meta_newkey(newkey); - rocksdb::Status s = db_->Get(default_read_options_, handles_[kSetsMetaCF], base_meta_key.Encode(), &meta_value); + rocksdb::Status s = db_->Get(default_read_options_, handles_[kMetaCF], base_meta_key.Encode(), &meta_value); if (s.ok()) { ParsedSetsMetaValue parsed_sets_meta_value(&meta_value); if (parsed_sets_meta_value.IsStale()) { @@ -1364,7 +1478,7 @@ Status Redis::SetsRenamenx(const Slice& key, Redis* new_inst, const Slice& newke } // check if newkey exists. std::string new_meta_value; - s = new_inst->GetDB()->Get(default_read_options_, handles_[kSetsMetaCF], base_meta_newkey.Encode(), + s = new_inst->GetDB()->Get(default_read_options_, handles_[kMetaCF], base_meta_newkey.Encode(), &new_meta_value); if (s.ok()) { ParsedSetsMetaValue parsed_sets_new_meta_value(&new_meta_value); @@ -1375,12 +1489,12 @@ Status Redis::SetsRenamenx(const Slice& key, Redis* new_inst, const Slice& newke // copy a new set with newkey statistic = parsed_sets_meta_value.Count(); - s = new_inst->GetDB()->Put(default_write_options_, handles_[kSetsMetaCF], base_meta_newkey.Encode(), meta_value); + s = new_inst->GetDB()->Put(default_write_options_, handles_[kMetaCF], base_meta_newkey.Encode(), meta_value); new_inst->UpdateSpecificKeyStatistics(DataType::kSets, newkey.ToString(), statistic); // SetsDel key parsed_sets_meta_value.InitialMetaValue(); - s = db_->Put(default_write_options_, handles_[kSetsMetaCF], base_meta_key.Encode(), meta_value); + s = db_->Put(default_write_options_, handles_[kMetaCF], base_meta_key.Encode(), meta_value); UpdateSpecificKeyStatistics(DataType::kSets, key.ToString(), statistic); } return s; @@ -1395,9 +1509,12 @@ void Redis::ScanSets() { auto current_time = static_cast(time(nullptr)); INFO("***************Sets Meta Data***************"); - auto meta_iter = db_->NewIterator(iterator_options, handles_[kSetsMetaCF]); + auto meta_iter = db_->NewIterator(iterator_options, handles_[kMetaCF]); for (meta_iter->SeekToFirst(); meta_iter->Valid(); meta_iter->Next()) { ParsedSetsMetaValue parsed_sets_meta_value(meta_iter->value()); + if (!parsed_sets_meta_value.IsSameType(Type::kSet)) { + continue; + } ParsedBaseMetaKey parsed_meta_key(meta_iter->key()); int32_t survival_time = 0; if (parsed_sets_meta_value.Etime() != 0) { diff --git a/src/storage/src/redis_strings.cc b/src/storage/src/redis_strings.cc index 14d25c81a..86f24878b 100644 --- a/src/storage/src/redis_strings.cc +++ b/src/storage/src/redis_strings.cc @@ -13,11 +13,11 @@ #include "src/redis.h" #include "src/scope_record_lock.h" #include "src/scope_snapshot.h" -#include "src/strings_filter.h" -#include "storage/storage_define.h" #include "storage/util.h" namespace storage { + +// TODO(dingxiaoshuai123) change Status Redis::ScanStringsKeyNum(KeyInfo* key_info) { uint64_t keys = 0; uint64_t expires = 0; @@ -57,6 +57,7 @@ Status Redis::ScanStringsKeyNum(KeyInfo* key_info) { return Status::OK(); } +// TODO(dingxiaoshuai123) change Status Redis::StringsPKPatternMatchDel(const std::string& pattern, int32_t* ret) { rocksdb::ReadOptions iterator_options; const rocksdb::Snapshot* snapshot; @@ -77,7 +78,7 @@ Status Redis::StringsPKPatternMatchDel(const std::string& pattern, int32_t* ret) ParsedStringsValue parsed_strings_value(&value); if (!parsed_strings_value.IsStale() && (StringMatch(pattern.data(), pattern.size(), key.data(), key.size(), 0) != 0)) { - batch.Delete(key); + batch.Delete(handles_[kMetaCF], key); } // In order to be more efficient, we use batch deletion here if (static_cast(batch.Count()) >= BATCH_DELETE_LIMIT) { @@ -105,6 +106,7 @@ Status Redis::StringsPKPatternMatchDel(const std::string& pattern, int32_t* ret) } Status Redis::Append(const Slice& key, const Slice& value, int32_t* ret) { + rocksdb::WriteBatch batch; std::string old_value; *ret = 0; ScopeRecordLock l(lock_mgr_, key); @@ -129,7 +131,8 @@ Status Redis::Append(const Slice& key, const Slice& value, int32_t* ret) { } else if (s.IsNotFound()) { *ret = static_cast(value.size()); StringsValue strings_value(value); - return db_->Put(default_write_options_, base_key.Encode(), strings_value.Encode()); + batch.Put(handles_[kMetaCF], base_key.Encode(), strings_value.Encode()); + return db_->Write(default_write_options_, &batch); } return s; } @@ -279,7 +282,9 @@ Status Redis::BitOp(BitOpType op, const std::string& dest_key, const std::vector StringsValue strings_value(Slice(dest_value.c_str(), max_len)); ScopeRecordLock l(lock_mgr_, dest_key); BaseKey base_dest_key(dest_key); - return db_->Put(default_write_options_, base_dest_key.Encode(), strings_value.Encode()); + rocksdb::WriteBatch batch; + batch.Put(handles_[kMetaCF], base_dest_key.Encode(), strings_value.Encode()); + return db_->Write(default_write_options_, &batch); } Status Redis::Decrby(const Slice& key, int64_t value, int64_t* ret) { @@ -318,7 +323,9 @@ Status Redis::Decrby(const Slice& key, int64_t value, int64_t* ret) { *ret = -value; new_value = std::to_string(*ret); StringsValue strings_value(new_value); - return db_->Put(default_write_options_, base_key.Encode(), strings_value.Encode()); + rocksdb::WriteBatch batch; + batch.Put(handles_[kMetaCF], base_key.Encode(), strings_value.Encode()); + return db_->Write(default_write_options_, &batch); } else { return s; } @@ -497,7 +504,9 @@ Status Redis::GetSet(const Slice& key, const Slice& value, std::string* old_valu return s; } StringsValue strings_value(value); - return db_->Put(default_write_options_, base_key.Encode(), strings_value.Encode()); + rocksdb::WriteBatch batch; + batch.Put(handles_[kMetaCF], base_key.Encode(), strings_value.Encode()); + return db_->Write(default_write_options_, &batch); } Status Redis::Incrby(const Slice& key, int64_t value, int64_t* ret) { @@ -536,7 +545,9 @@ Status Redis::Incrby(const Slice& key, int64_t value, int64_t* ret) { *ret = value; Int64ToStr(buf, 32, value); StringsValue strings_value(buf); - return db_->Put(default_write_options_, base_key.Encode(), strings_value.Encode()); + rocksdb::WriteBatch batch; + batch.Put(handles_[kMetaCF], base_key.Encode(), strings_value.Encode()); + return db_->Write(default_write_options_, &batch); } else { return s; } @@ -581,7 +592,9 @@ Status Redis::Incrbyfloat(const Slice& key, const Slice& value, std::string* ret LongDoubleToStr(long_double_by, &new_value); *ret = new_value; StringsValue strings_value(new_value); - return db_->Put(default_write_options_, base_key.Encode(), strings_value.Encode()); + rocksdb::WriteBatch batch; + batch.Put(handles_[kMetaCF], base_key.Encode(), strings_value.Encode()); + return db_->Write(default_write_options_, &batch); } else { return s; } @@ -599,7 +612,7 @@ Status Redis::MSet(const std::vector& kvs) { for (const auto& kv : kvs) { BaseKey base_key(kv.key); StringsValue strings_value(kv.value); - batch->Put(kStringsCF, base_key.Encode(), strings_value.Encode()); + batch->Put(kMetaCF, base_key.Encode(), strings_value.Encode()); } return batch->Commit(); } @@ -633,9 +646,8 @@ Status Redis::Set(const Slice& key, const Slice& value) { StringsValue strings_value(value); auto batch = Batch::CreateBatch(this); ScopeRecordLock l(lock_mgr_, key); - BaseKey base_key(key); - batch->Put(kStringsCF, base_key.Encode(), strings_value.Encode()); + batch->Put(kMetaCF, base_key.Encode(), strings_value.Encode()); return batch->Commit(); } @@ -665,7 +677,9 @@ Status Redis::Setxx(const Slice& key, const Slice& value, int32_t* ret, const ui if (ttl > 0) { strings_value.SetRelativeTimestamp(ttl); } - return db_->Put(default_write_options_, base_key.Encode(), strings_value.Encode()); + rocksdb::WriteBatch batch; + batch.Put(handles_[kMetaCF], base_key.Encode(), strings_value.Encode()); + return db_->Write(default_write_options_, &batch); } } @@ -713,7 +727,7 @@ Status Redis::SetBit(const Slice& key, int64_t offset, int32_t on, int32_t* ret) StringsValue strings_value(data_value); strings_value.SetEtime(timestamp); auto batch = Batch::CreateBatch(this); - batch->Put(kStringsCF, base_key.Encode(), strings_value.Encode()); + batch->Put(kMetaCF, base_key.Encode(), strings_value.Encode()); return batch->Commit(); } else { return s; @@ -733,7 +747,7 @@ Status Redis::Setex(const Slice& key, const Slice& value, uint64_t ttl) { BaseKey base_key(key); ScopeRecordLock l(lock_mgr_, key); auto batch = Batch::CreateBatch(this); - batch->Put(kStringsCF, base_key.Encode(), strings_value.Encode()); + batch->Put(kMetaCF, base_key.Encode(), strings_value.Encode()); return batch->Commit(); } @@ -761,7 +775,9 @@ Status Redis::Setnx(const Slice& key, const Slice& value, int32_t* ret, const ui if (ttl > 0) { strings_value.SetRelativeTimestamp(ttl); } - s = db_->Put(default_write_options_, base_key.Encode(), strings_value.Encode()); + rocksdb::WriteBatch batch; + batch.Put(handles_[kMetaCF], base_key.Encode(), strings_value.Encode()); + db_->Write(default_write_options_, &batch); if (s.ok()) { *ret = 1; } @@ -818,7 +834,9 @@ Status Redis::Delvx(const Slice& key, const Slice& value, int32_t* ret) { } else { if (value.compare(parsed_strings_value.UserValue()) == 0) { *ret = 1; - return db_->Delete(default_write_options_, base_key.Encode()); + rocksdb::WriteBatch batch; + batch.Delete(handles_[kMetaCF], base_key.Encode()); + return db_->Write(default_write_options_, &batch); } else { *ret = -1; } @@ -871,7 +889,9 @@ Status Redis::Setrange(const Slice& key, int64_t start_offset, const Slice& valu new_value = tmp.append(value.data()); *ret = static_cast(new_value.length()); StringsValue strings_value(new_value); - return db_->Put(default_write_options_, base_key.Encode(), strings_value.Encode()); + rocksdb::WriteBatch batch; + batch.Put(handles_[kMetaCF], base_key.Encode(), strings_value.Encode()); + return db_->Write(default_write_options_, &batch); } return s; } @@ -1086,7 +1106,9 @@ Status Redis::PKSetexAt(const Slice& key, const Slice& value, uint64_t timestamp BaseKey base_key(key); ScopeRecordLock l(lock_mgr_, key); strings_value.SetEtime(uint64_t(timestamp)); - return db_->Put(default_write_options_, base_key.Encode(), strings_value.Encode()); + rocksdb::WriteBatch batch; + batch.Put(handles_[kMetaCF], base_key.Encode(), strings_value.Encode()); + return db_->Write(default_write_options_, &batch); } Status Redis::StringsExpire(const Slice& key, uint64_t ttl) { @@ -1104,7 +1126,9 @@ Status Redis::StringsExpire(const Slice& key, uint64_t ttl) { parsed_strings_value.SetRelativeTimestamp(ttl); return db_->Put(default_write_options_, base_key.Encode(), value); } else { - return db_->Delete(default_write_options_, base_key.Encode()); + rocksdb::WriteBatch batch; + batch.Delete(handles_[kMetaCF], base_key.Encode()); + return db_->Write(default_write_options_, &batch); } } return s; @@ -1121,7 +1145,9 @@ Status Redis::StringsDel(const Slice& key) { if (parsed_strings_value.IsStale()) { return Status::NotFound("Stale"); } - return db_->Delete(default_write_options_, base_key.Encode()); + rocksdb::WriteBatch batch; + batch.Delete(handles_[kMetaCF], base_key.Encode()); + return db_->Write(default_write_options_, &batch); } return s; } @@ -1141,17 +1167,93 @@ Status Redis::StringsExpireat(const Slice& key, uint64_t timestamp) { parsed_strings_value.SetEtime(uint64_t(timestamp)); return db_->Put(default_write_options_, base_key.Encode(), value); } else { - return db_->Delete(default_write_options_, base_key.Encode()); + rocksdb::WriteBatch batch; + batch.Delete(handles_[kMetaCF], base_key.Encode()); + return db_->Write(default_write_options_, &batch); } } } return s; } +rocksdb::Status Redis::Persist(const Slice& key) { + std::string meta_value; + BaseMetaKey base_meta_key(key); + rocksdb::Status s = db_->Get(default_read_options_, handles_[kMetaCF], base_meta_key.Encode(), &meta_value); + if (s.ok()) { + auto type = static_cast(static_cast(meta_value[0])); + if (type == Type::kSet) { + return SetsPersist(key); + } else if (type == Type::kZset) { + return ZsetsPersist(key); + } else if (type == Type::kHash) { + return HashesPersist(key); + } else if (type == Type::kList) { + return ListsPersist(key); + } else { + return StringsPersist(key); + } + } + return rocksdb::Status::NotFound(); +} + +rocksdb::Status Redis::TTL(const Slice& key, uint64_t* timestamp) { + std::string meta_value; + BaseMetaKey base_meta_key(key); + rocksdb::Status s = db_->Get(default_read_options_, handles_[kMetaCF], base_meta_key.Encode(), &meta_value); + if (s.ok()) { + auto type = static_cast(static_cast(meta_value[0])); + if (type == Type::kSet) { + return SetsTTL(key, timestamp); + } else if (type == Type::kZset) { + return ZsetsTTL(key, timestamp); + } else if (type == Type::kHash) { + return HashesTTL(key, timestamp); + } else if (type == Type::kList) { + return ListsTTL(key, timestamp); + } else { + return StringsTTL(key, timestamp); + } + } + return rocksdb::Status::NotFound(); +} + +rocksdb::Status Redis::GetType(const storage::Slice& key, std::string& types) { + std::string meta_value; + BaseMetaKey base_meta_key(key); + rocksdb::Status s = db_->Get(default_read_options_, handles_[kMetaCF], base_meta_key.Encode(), &meta_value); + if (s.ok()) { + auto type = static_cast(static_cast(meta_value[0])); + if (type == Type::kSet) { + types = "set"; + } else if (type == Type::kZset) { + types = "zset"; + } else if (type == Type::kHash) { + types = "hash"; + } else if (type == Type::kList) { + types = "list"; + } else { + types = "string"; + } + return Status::OK(); + } + types = "none"; + return rocksdb::Status::NotFound(); +} + +rocksdb::Status Redis::IsExist(const storage::Slice& key) { + std::string meta_value; + BaseMetaKey base_meta_key(key); + rocksdb::Status s = db_->Get(default_read_options_, handles_[kMetaCF], base_meta_key.Encode(), &meta_value); + if (s.ok()) { + return Status::OK(); + } + return rocksdb::Status::NotFound(); +} + Status Redis::StringsPersist(const Slice& key) { std::string value; ScopeRecordLock l(lock_mgr_, key); - BaseKey base_key(key); Status s = db_->Get(default_read_options_, base_key.Encode(), &value); if (s.ok()) { @@ -1246,6 +1348,78 @@ Status Redis::StringsRenamenx(const Slice& key, Redis* new_inst, const Slice& ne return s; } + rocksdb::Status Redis::Del(const Slice& key) { + std::string meta_value; + BaseMetaKey base_meta_key(key); + rocksdb::Status s = db_->Get(default_read_options_, handles_[kMetaCF], base_meta_key.Encode(), &meta_value); + if (s.ok()) { + auto type = static_cast(static_cast(meta_value[0])); + switch (type) { + case DataType::kSets: + return SetsDel(key); + case DataType::kZSets: + return ZsetsDel(key); + case DataType::kHashes: + return HashesDel(key); + case DataType::kLists: + return ListsDel(key); + case DataType::kStrings: + return StringsDel(key); + default: + return rocksdb::Status::NotFound(); + } + } + return rocksdb::Status::NotFound(); + } + rocksdb::Status Redis::Expire(const Slice& key, uint64_t ttl) { + std::string meta_value; + BaseMetaKey base_meta_key(key); + rocksdb::Status s = db_->Get(default_read_options_, handles_[kMetaCF], base_meta_key.Encode(), &meta_value); + if (s.ok()) { + auto type = static_cast(static_cast(meta_value[0])); + switch (type) { + case DataType::kSets: + return SetsExpire(key, ttl); + case DataType::kZSets: + return ZsetsExpire(key, ttl); + case DataType::kHashes: + return HashesExpire(key, ttl); + case DataType::kLists: + return ListsExpire(key, ttl); + case DataType::kStrings: + return StringsExpire(key, ttl); + default: + return rocksdb::Status::NotFound(); + } + } + return rocksdb::Status::NotFound(); + } + rocksdb::Status Redis::Exists(const Slice& key) { + std::string meta_value; + uint64_t llen = 0; + std::string value; + int32_t ret = 0; + BaseMetaKey base_meta_key(key); + rocksdb::Status s = db_->Get(default_read_options_, handles_[kMetaCF], base_meta_key.Encode(), &meta_value); + if (s.ok()) { + auto type = static_cast(static_cast(meta_value[0])); + switch (type) { + case DataType::kSets: + return SCard(key, &ret); + case DataType::kZSets: + return ZCard(key, &ret); + case DataType::kHashes: + return HLen(key, &ret); + case DataType::kLists: + return LLen(key, &llen); + case DataType::kStrings: + return Get(key, &value); + default: + return rocksdb::Status::NotFound(); + } + } + return rocksdb::Status::NotFound(); + } void Redis::ScanStrings() { rocksdb::ReadOptions iterator_options; const rocksdb::Snapshot* snapshot; diff --git a/src/storage/src/redis_zsets.cc b/src/storage/src/redis_zsets.cc index 44e9afc8e..5d2b7cd26 100644 --- a/src/storage/src/redis_zsets.cc +++ b/src/storage/src/redis_zsets.cc @@ -37,9 +37,12 @@ Status Redis::ScanZsetsKeyNum(KeyInfo* key_info) { int64_t curtime; rocksdb::Env::Default()->GetCurrentTime(&curtime); - rocksdb::Iterator* iter = db_->NewIterator(iterator_options, handles_[kZsetsMetaCF]); + rocksdb::Iterator* iter = db_->NewIterator(iterator_options, handles_[kMetaCF]); for (iter->SeekToFirst(); iter->Valid(); iter->Next()) { ParsedZSetsMetaValue parsed_zsets_meta_value(iter->value()); + if (!parsed_zsets_meta_value.IsSameType(Type::kZset)) { + continue; + } if (parsed_zsets_meta_value.IsStale() || parsed_zsets_meta_value.Count() == 0) { invaild_keys++; } else { @@ -71,16 +74,19 @@ Status Redis::ZsetsPKPatternMatchDel(const std::string& pattern, int32_t* ret) { int32_t total_delete = 0; Status s; rocksdb::WriteBatch batch; - rocksdb::Iterator* iter = db_->NewIterator(iterator_options, handles_[kZsetsMetaCF]); + rocksdb::Iterator* iter = db_->NewIterator(iterator_options, handles_[kMetaCF]); iter->SeekToFirst(); while (iter->Valid()) { ParsedBaseMetaKey meta_key(iter->key().ToString()); meta_value = iter->value().ToString(); ParsedZSetsMetaValue parsed_zsets_meta_value(&meta_value); + if (!parsed_zsets_meta_value.IsSameType(Type::kZset)) { + continue; + } if (!parsed_zsets_meta_value.IsStale() && (parsed_zsets_meta_value.Count() != 0) && (StringMatch(pattern.data(), pattern.size(), meta_key.Key().data(), meta_key.Key().size(), 0) != 0)) { parsed_zsets_meta_value.InitialMetaValue(); - batch.Put(handles_[kZsetsMetaCF], key, meta_value); + batch.Put(handles_[kMetaCF], key, meta_value); } if (static_cast(batch.Count()) >= BATCH_DELETE_LIMIT) { s = db_->Write(default_write_options_, &batch); @@ -114,8 +120,12 @@ Status Redis::ZPopMax(const Slice& key, const int64_t count, std::vectorGet(default_read_options_, handles_[kZsetsMetaCF], base_meta_key.Encode(), &meta_value); + Status s = db_->Get(default_read_options_, handles_[kMetaCF], base_meta_key.Encode(), &meta_value); if (s.ok()) { + auto type = static_cast(static_cast(meta_value[0])); + if (type != Type::kZset) { + return Status::InvalidArgument("WRONGTYPE Operation against a key holding the wrong kind of value"); + } ParsedZSetsMetaValue parsed_zsets_meta_value(&meta_value); if (parsed_zsets_meta_value.IsStale()) { return Status::NotFound("Stale"); @@ -144,7 +154,7 @@ Status Redis::ZPopMax(const Slice& key, const int64_t count, std::vectorWrite(default_write_options_, &batch); UpdateSpecificKeyStatistics(DataType::kZSets, key.ToString(), statistic); return s; @@ -162,8 +172,12 @@ Status Redis::ZPopMin(const Slice& key, const int64_t count, std::vectorGet(default_read_options_, handles_[kZsetsMetaCF], base_meta_key.Encode(), &meta_value); + Status s = db_->Get(default_read_options_, handles_[kMetaCF], base_meta_key.Encode(), &meta_value); if (s.ok()) { + auto type = static_cast(static_cast(meta_value[0])); + if (type != Type::kZset) { + return Status::InvalidArgument("WRONGTYPE Operation against a key holding the wrong kind of value"); + } ParsedZSetsMetaValue parsed_zsets_meta_value(&meta_value); if (parsed_zsets_meta_value.IsStale()) { return Status::NotFound("Stale"); @@ -192,7 +206,7 @@ Status Redis::ZPopMin(const Slice& key, const int64_t count, std::vectorWrite(default_write_options_, &batch); UpdateSpecificKeyStatistics(DataType::kZSets, key.ToString(), statistic); return s; @@ -221,9 +235,13 @@ Status Redis::ZAdd(const Slice& key, const std::vector& score_membe ScopeRecordLock l(lock_mgr_, key); BaseMetaKey base_meta_key(key); - Status s = db_->Get(default_read_options_, handles_[kZsetsMetaCF], base_meta_key.Encode(), &meta_value); + Status s = db_->Get(default_read_options_, handles_[kMetaCF], base_meta_key.Encode(), &meta_value); if (s.ok()) { bool vaild = true; + auto type = static_cast(static_cast(meta_value[0])); + if (type != Type::kZset) { + return Status::InvalidArgument("WRONGTYPE Operation against a key holding the wrong kind of value"); + } ParsedZSetsMetaValue parsed_zsets_meta_value(&meta_value); if (parsed_zsets_meta_value.IsStale() || parsed_zsets_meta_value.Count() == 0) { vaild = false; @@ -277,14 +295,14 @@ Status Redis::ZAdd(const Slice& key, const std::vector& score_membe return Status::InvalidArgument("zset size overflow"); } parsed_zsets_meta_value.ModifyCount(cnt); - batch->Put(kZsetsMetaCF, base_meta_key.Encode(), meta_value); + batch->Put(kMetaCF, base_meta_key.Encode(), meta_value); *ret = cnt; } else if (s.IsNotFound()) { - char buf[4]; - EncodeFixed32(buf, filtered_score_members.size()); - ZSetsMetaValue zsets_meta_value(Slice(buf, sizeof(int32_t))); + char str[4]; + EncodeFixed32(str, filtered_score_members.size()); + ZSetsMetaValue zsets_meta_value(Type::kZset, Slice(str, 4)); version = zsets_meta_value.UpdateVersion(); - batch->Put(kZsetsMetaCF, base_meta_key.Encode(), zsets_meta_value.Encode()); + batch->Put(kMetaCF, base_meta_key.Encode(), zsets_meta_value.Encode()); for (const auto& sm : filtered_score_members) { ZSetsMemberKey zsets_member_key(key, version, sm.member); const void* ptr_score = reinterpret_cast(&sm.score); @@ -310,8 +328,12 @@ Status Redis::ZCard(const Slice& key, int32_t* card) { std::string meta_value; BaseMetaKey base_meta_key(key); - Status s = db_->Get(default_read_options_, handles_[kZsetsMetaCF], base_meta_key.Encode(), &meta_value); + Status s = db_->Get(default_read_options_, handles_[kMetaCF], base_meta_key.Encode(), &meta_value); if (s.ok()) { + auto type = static_cast(static_cast(meta_value[0])); + if (type != Type::kZset) { + return Status::InvalidArgument("WRONGTYPE Operation against a key holding the wrong kind of value"); + } ParsedZSetsMetaValue parsed_zsets_meta_value(&meta_value); if (parsed_zsets_meta_value.IsStale()) { *card = 0; @@ -336,8 +358,12 @@ Status Redis::ZCount(const Slice& key, double min, double max, bool left_close, read_options.snapshot = snapshot; BaseMetaKey base_meta_key(key); - Status s = db_->Get(read_options, handles_[kZsetsMetaCF], base_meta_key.Encode(), &meta_value); + Status s = db_->Get(read_options, handles_[kMetaCF], base_meta_key.Encode(), &meta_value); if (s.ok()) { + auto type = static_cast(static_cast(meta_value[0])); + if (type != Type::kZset) { + return Status::InvalidArgument("WRONGTYPE Operation against a key holding the wrong kind of value"); + } ParsedZSetsMetaValue parsed_zsets_meta_value(&meta_value); if (parsed_zsets_meta_value.IsStale()) { return Status::NotFound("Stale"); @@ -394,8 +420,12 @@ Status Redis::ZIncrby(const Slice& key, const Slice& member, double increment, d ScopeRecordLock l(lock_mgr_, key); BaseMetaKey base_meta_key(key); - Status s = db_->Get(default_read_options_, handles_[kZsetsMetaCF], base_meta_key.Encode(), &meta_value); + Status s = db_->Get(default_read_options_, handles_[kMetaCF], base_meta_key.Encode(), &meta_value); if (s.ok()) { + auto type = static_cast(static_cast(meta_value[0])); + if (type != Type::kZset) { + return Status::InvalidArgument("WRONGTYPE Operation against a key holding the wrong kind of value"); + } ParsedZSetsMetaValue parsed_zsets_meta_value(&meta_value); if (parsed_zsets_meta_value.IsStale() || parsed_zsets_meta_value.Count() == 0) { version = parsed_zsets_meta_value.InitialMetaValue(); @@ -423,16 +453,17 @@ Status Redis::ZIncrby(const Slice& key, const Slice& member, double increment, d return Status::InvalidArgument("zset size overflow"); } parsed_zsets_meta_value.ModifyCount(1); - batch.Put(handles_[kZsetsMetaCF], base_meta_key.Encode(), meta_value); + batch.Put(handles_[kMetaCF], base_meta_key.Encode(), meta_value); } else { return s; } } else if (s.IsNotFound()) { - char buf[8]; - EncodeFixed32(buf, 1); - ZSetsMetaValue zsets_meta_value(Slice(buf, sizeof(int32_t))); + char str[4]; + EncodeFixed32(str, 1); + ZSetsMetaValue zsets_meta_value(Type::kZset, Slice(str, 4)); + version = zsets_meta_value.UpdateVersion(); - batch.Put(handles_[kZsetsMetaCF], base_meta_key.Encode(), zsets_meta_value.Encode()); + batch.Put(handles_[kMetaCF], base_meta_key.Encode(), zsets_meta_value.Encode()); score = increment; } else { return s; @@ -462,8 +493,12 @@ Status Redis::ZRange(const Slice& key, int32_t start, int32_t stop, std::vector< read_options.snapshot = snapshot; BaseMetaKey base_meta_key(key); - Status s = db_->Get(read_options, handles_[kZsetsMetaCF], base_meta_key.Encode(), &meta_value); + Status s = db_->Get(read_options, handles_[kMetaCF], base_meta_key.Encode(), &meta_value); if (s.ok()) { + auto type = static_cast(static_cast(meta_value[0])); + if (type != Type::kZset) { + return Status::InvalidArgument("WRONGTYPE Operation against a key holding the wrong kind of value"); + } ParsedZSetsMetaValue parsed_zsets_meta_value(&meta_value); if (parsed_zsets_meta_value.IsStale()) { return Status::NotFound("Stale"); @@ -510,8 +545,12 @@ Status Redis::ZRangeWithTTL(const Slice& key, int32_t start, int32_t stop, std:: read_options.snapshot = snapshot; BaseMetaKey base_meta_key(key); - Status s = db_->Get(read_options, handles_[kZsetsMetaCF], base_meta_key.Encode(), &meta_value); + Status s = db_->Get(read_options, handles_[kMetaCF], base_meta_key.Encode(), &meta_value); if (s.ok()) { + auto type = static_cast(static_cast(meta_value[0])); + if (type != Type::kZset) { + return Status::InvalidArgument("WRONGTYPE Operation against a key holding the wrong kind of value"); + } ParsedZSetsMetaValue parsed_zsets_meta_value(&meta_value); if (parsed_zsets_meta_value.Count() == 0) { return Status::NotFound(); @@ -567,8 +606,12 @@ Status Redis::ZRangebyscore(const Slice& key, double min, double max, bool left_ read_options.snapshot = snapshot; BaseMetaKey base_meta_key(key); - Status s = db_->Get(read_options, handles_[kZsetsMetaCF], base_meta_key.Encode(), &meta_value); + Status s = db_->Get(read_options, handles_[kMetaCF], base_meta_key.Encode(), &meta_value); if (s.ok()) { + auto type = static_cast(static_cast(meta_value[0])); + if (type != Type::kZset) { + return Status::InvalidArgument("WRONGTYPE Operation against a key holding the wrong kind of value"); + } ParsedZSetsMetaValue parsed_zsets_meta_value(&meta_value); if (parsed_zsets_meta_value.IsStale()) { return Status::NotFound("Stale"); @@ -634,8 +677,12 @@ Status Redis::ZRank(const Slice& key, const Slice& member, int32_t* rank) { read_options.snapshot = snapshot; BaseMetaKey base_meta_key(key); - Status s = db_->Get(read_options, handles_[kZsetsMetaCF], base_meta_key.Encode(), &meta_value); + Status s = db_->Get(read_options, handles_[kMetaCF], base_meta_key.Encode(), &meta_value); if (s.ok()) { + auto type = static_cast(static_cast(meta_value[0])); + if (type != Type::kZset) { + return Status::InvalidArgument("WRONGTYPE Operation against a key holding the wrong kind of value"); + } ParsedZSetsMetaValue parsed_zsets_meta_value(&meta_value); if (parsed_zsets_meta_value.IsStale()) { return Status::NotFound("Stale"); @@ -686,8 +733,12 @@ Status Redis::ZRem(const Slice& key, const std::vector& members, in ScopeRecordLock l(lock_mgr_, key); BaseMetaKey base_meta_key(key); - Status s = db_->Get(default_read_options_, handles_[kZsetsMetaCF], base_meta_key.Encode(), &meta_value); + Status s = db_->Get(default_read_options_, handles_[kMetaCF], base_meta_key.Encode(), &meta_value); if (s.ok()) { + auto type = static_cast(static_cast(meta_value[0])); + if (type != Type::kZset) { + return Status::InvalidArgument("WRONGTYPE Operation against a key holding the wrong kind of value"); + } ParsedZSetsMetaValue parsed_zsets_meta_value(&meta_value); if (parsed_zsets_meta_value.IsStale()) { return Status::NotFound("Stale"); @@ -721,7 +772,7 @@ Status Redis::ZRem(const Slice& key, const std::vector& members, in return Status::InvalidArgument("zset size overflow"); } parsed_zsets_meta_value.ModifyCount(-del_cnt); - batch.Put(handles_[kZsetsMetaCF], base_meta_key.Encode(), meta_value); + batch.Put(handles_[kMetaCF], base_meta_key.Encode(), meta_value); } } else { return s; @@ -739,8 +790,12 @@ Status Redis::ZRemrangebyrank(const Slice& key, int32_t start, int32_t stop, int ScopeRecordLock l(lock_mgr_, key); BaseMetaKey base_meta_key(key); - Status s = db_->Get(default_read_options_, handles_[kZsetsMetaCF], base_meta_key.Encode(), &meta_value); + Status s = db_->Get(default_read_options_, handles_[kMetaCF], base_meta_key.Encode(), &meta_value); if (s.ok()) { + auto type = static_cast(static_cast(meta_value[0])); + if (type != Type::kZset) { + return Status::InvalidArgument("WRONGTYPE Operation against a key holding the wrong kind of value"); + } ParsedZSetsMetaValue parsed_zsets_meta_value(&meta_value); if (parsed_zsets_meta_value.IsStale()) { return Status::NotFound("Stale"); @@ -778,7 +833,7 @@ Status Redis::ZRemrangebyrank(const Slice& key, int32_t start, int32_t stop, int return Status::InvalidArgument("zset size overflow"); } parsed_zsets_meta_value.ModifyCount(-del_cnt); - batch.Put(handles_[kZsetsMetaCF], base_meta_key.Encode(), meta_value); + batch.Put(handles_[kMetaCF], base_meta_key.Encode(), meta_value); } } else { return s; @@ -797,8 +852,12 @@ Status Redis::ZRemrangebyscore(const Slice& key, double min, double max, bool le ScopeRecordLock l(lock_mgr_, key); BaseMetaKey base_meta_key(key); - Status s = db_->Get(default_read_options_, handles_[kZsetsMetaCF], base_meta_key.Encode(), &meta_value); + Status s = db_->Get(default_read_options_, handles_[kMetaCF], base_meta_key.Encode(), &meta_value); if (s.ok()) { + auto type = static_cast(static_cast(meta_value[0])); + if (type != Type::kZset) { + return Status::InvalidArgument("WRONGTYPE Operation against a key holding the wrong kind of value"); + } ParsedZSetsMetaValue parsed_zsets_meta_value(&meta_value); if (parsed_zsets_meta_value.IsStale()) { return Status::NotFound("Stale"); @@ -848,7 +907,7 @@ Status Redis::ZRemrangebyscore(const Slice& key, double min, double max, bool le return Status::InvalidArgument("zset size overflow"); } parsed_zsets_meta_value.ModifyCount(-del_cnt); - batch.Put(handles_[kZsetsMetaCF], base_meta_key.Encode(), meta_value); + batch.Put(handles_[kMetaCF], base_meta_key.Encode(), meta_value); } } else { return s; @@ -868,8 +927,12 @@ Status Redis::ZRevrange(const Slice& key, int32_t start, int32_t stop, std::vect read_options.snapshot = snapshot; BaseMetaKey base_meta_key(key); - Status s = db_->Get(read_options, handles_[kZsetsMetaCF], base_meta_key.Encode(), &meta_value); + Status s = db_->Get(read_options, handles_[kMetaCF], base_meta_key.Encode(), &meta_value); if (s.ok()) { + auto type = static_cast(static_cast(meta_value[0])); + if (type != Type::kZset) { + return Status::InvalidArgument("WRONGTYPE Operation against a key holding the wrong kind of value"); + } ParsedZSetsMetaValue parsed_zsets_meta_value(&meta_value); if (parsed_zsets_meta_value.IsStale()) { return Status::NotFound("Stale"); @@ -916,8 +979,12 @@ Status Redis::ZRevrangebyscore(const Slice& key, double min, double max, bool le read_options.snapshot = snapshot; BaseMetaKey base_meta_key(key); - Status s = db_->Get(read_options, handles_[kZsetsMetaCF], base_meta_key.Encode(), &meta_value); + Status s = db_->Get(read_options, handles_[kMetaCF], base_meta_key.Encode(), &meta_value); if (s.ok()) { + auto type = static_cast(static_cast(meta_value[0])); + if (type != Type::kZset) { + return Status::InvalidArgument("WRONGTYPE Operation against a key holding the wrong kind of value"); + } ParsedZSetsMetaValue parsed_zsets_meta_value(&meta_value); if (parsed_zsets_meta_value.IsStale()) { return Status::NotFound("Stale"); @@ -982,8 +1049,12 @@ Status Redis::ZRevrank(const Slice& key, const Slice& member, int32_t* rank) { read_options.snapshot = snapshot; BaseMetaKey base_meta_key(key); - Status s = db_->Get(read_options, handles_[kZsetsMetaCF], base_meta_key.Encode(), &meta_value); + Status s = db_->Get(read_options, handles_[kMetaCF], base_meta_key.Encode(), &meta_value); if (s.ok()) { + auto type = static_cast(static_cast(meta_value[0])); + if (type != Type::kZset) { + return Status::InvalidArgument("WRONGTYPE Operation against a key holding the wrong kind of value"); + } ParsedZSetsMetaValue parsed_zsets_meta_value(&meta_value); if (parsed_zsets_meta_value.IsStale()) { return Status::NotFound("Stale"); @@ -1025,8 +1096,12 @@ Status Redis::ZScore(const Slice& key, const Slice& member, double* score) { read_options.snapshot = snapshot; BaseMetaKey base_meta_key(key); - Status s = db_->Get(read_options, handles_[kZsetsMetaCF], base_meta_key.Encode(), &meta_value); + Status s = db_->Get(read_options, handles_[kMetaCF], base_meta_key.Encode(), &meta_value); if (s.ok()) { + auto type = static_cast(static_cast(meta_value[0])); + if (type != Type::kZset) { + return Status::InvalidArgument("WRONGTYPE Operation against a key holding the wrong kind of value"); + } ParsedZSetsMetaValue parsed_zsets_meta_value(&meta_value); uint64_t version = parsed_zsets_meta_value.Version(); if (parsed_zsets_meta_value.IsStale()) { @@ -1062,8 +1137,12 @@ Status Redis::ZGetAll(const Slice& key, double weight, std::mapGet(read_options, handles_[kZsetsMetaCF], base_meta_key.Encode(), &meta_value); + s = db_->Get(read_options, handles_[kMetaCF], base_meta_key.Encode(), &meta_value); if (s.ok()) { + auto type = static_cast(static_cast(meta_value[0])); + if (type != Type::kZset) { + return Status::InvalidArgument("WRONGTYPE Operation against a key holding the wrong kind of value"); + } ParsedZSetsMetaValue parsed_zsets_meta_value(&meta_value); if (!parsed_zsets_meta_value.IsStale() && parsed_zsets_meta_value.Count() != 0) { int32_t cur_index = 0; @@ -1105,9 +1184,12 @@ Status Redis::ZUnionstore(const Slice& destination, const std::vectorGet(read_options, handles_[kZsetsMetaCF], base_meta_key.Encode(), &meta_value); + s = db_->Get(read_options, handles_[kMetaCF], base_meta_key.Encode(), &meta_value); if (s.ok()) { ParsedZSetsMetaValue parsed_zsets_meta_value(&meta_value); + if (!parsed_zsets_meta_value.IsSameType(Type::kZset)) { + return Status::InvalidArgument("WRONGTYPE Operation against a key holding the wrong kind of value"); + } if (!parsed_zsets_meta_value.IsStale() && parsed_zsets_meta_value.Count() != 0) { int32_t cur_index = 0; int32_t stop_index = parsed_zsets_meta_value.Count() - 1; @@ -1149,8 +1231,12 @@ Status Redis::ZUnionstore(const Slice& destination, const std::vectorGet(read_options, handles_[kZsetsMetaCF], base_destination.Encode(), &meta_value); + s = db_->Get(read_options, handles_[kMetaCF], base_destination.Encode(), &meta_value); if (s.ok()) { + auto type = static_cast(static_cast(meta_value[0])); + if (type != Type::kZset) { + return Status::InvalidArgument("WRONGTYPE Operation against a key holding the wrong kind of value"); + } ParsedZSetsMetaValue parsed_zsets_meta_value(&meta_value); statistic = parsed_zsets_meta_value.Count(); version = parsed_zsets_meta_value.InitialMetaValue(); @@ -1158,13 +1244,13 @@ Status Redis::ZUnionstore(const Slice& destination, const std::vector(member_score_map.size())); - batch.Put(handles_[kZsetsMetaCF], base_destination.Encode(), meta_value); + batch.Put(handles_[kMetaCF], base_destination.Encode(), meta_value); } else { - char buf[4]; - EncodeFixed32(buf, member_score_map.size()); - ZSetsMetaValue zsets_meta_value(Slice(buf, sizeof(int32_t))); + char str[4]; + EncodeFixed32(str, member_score_map.size()); + ZSetsMetaValue zsets_meta_value(Type::kZset, Slice(str, 4)); version = zsets_meta_value.UpdateVersion(); - batch.Put(handles_[kZsetsMetaCF], base_destination.Encode(), zsets_meta_value.Encode()); + batch.Put(handles_[kMetaCF], base_destination.Encode(), zsets_meta_value.Encode()); } char score_buf[8]; @@ -1216,9 +1302,12 @@ Status Redis::ZInterstore(const Slice& destination, const std::vectorGet(read_options, handles_[kZsetsMetaCF], base_meta_key.Encode(), &meta_value); + s = db_->Get(read_options, handles_[kMetaCF], base_meta_key.Encode(), &meta_value); if (s.ok()) { ParsedZSetsMetaValue parsed_zsets_meta_value(&meta_value); + if (!parsed_zsets_meta_value.IsSameType(Type::kZset)) { + return Status::InvalidArgument("WRONGTYPE Operation against a key holding the wrong kind of value"); + } if (parsed_zsets_meta_value.IsStale() || parsed_zsets_meta_value.Count() == 0) { have_invalid_zsets = true; } else { @@ -1287,8 +1376,12 @@ Status Redis::ZInterstore(const Slice& destination, const std::vectorGet(read_options, handles_[kZsetsMetaCF], base_destination.Encode(), &meta_value); + s = db_->Get(read_options, handles_[kMetaCF], base_destination.Encode(), &meta_value); if (s.ok()) { + auto type = static_cast(static_cast(meta_value[0])); + if (type != Type::kZset) { + return Status::InvalidArgument("WRONGTYPE Operation against a key holding the wrong kind of value"); + } ParsedZSetsMetaValue parsed_zsets_meta_value(&meta_value); statistic = parsed_zsets_meta_value.Count(); version = parsed_zsets_meta_value.InitialMetaValue(); @@ -1296,13 +1389,13 @@ Status Redis::ZInterstore(const Slice& destination, const std::vector(final_score_members.size())); - batch.Put(handles_[kZsetsMetaCF], base_destination.Encode(), meta_value); + batch.Put(handles_[kMetaCF], base_destination.Encode(), meta_value); } else { - char buf[4]; - EncodeFixed32(buf, final_score_members.size()); - ZSetsMetaValue zsets_meta_value(Slice(buf, sizeof(int32_t))); + char str[4]; + EncodeFixed32(str, final_score_members.size()); + ZSetsMetaValue zsets_meta_value(Type::kZset, Slice(str, 4)); version = zsets_meta_value.UpdateVersion(); - batch.Put(handles_[kZsetsMetaCF], base_destination.Encode(), zsets_meta_value.Encode()); + batch.Put(handles_[kMetaCF], base_destination.Encode(), zsets_meta_value.Encode()); } char score_buf[8]; for (const auto& sm : final_score_members) { @@ -1338,8 +1431,12 @@ Status Redis::ZRangebylex(const Slice& key, const Slice& min, const Slice& max, bool right_not_limit = max.compare("+") == 0; BaseMetaKey base_meta_key(key); - Status s = db_->Get(read_options, handles_[kZsetsMetaCF], base_meta_key.Encode(), &meta_value); + Status s = db_->Get(read_options, handles_[kMetaCF], base_meta_key.Encode(), &meta_value); if (s.ok()) { + auto type = static_cast(static_cast(meta_value[0])); + if (type != Type::kZset) { + return Status::InvalidArgument("WRONGTYPE Operation against a key holding the wrong kind of value"); + } ParsedZSetsMetaValue parsed_zsets_meta_value(&meta_value); if (parsed_zsets_meta_value.IsStale() || parsed_zsets_meta_value.Count() == 0) { return Status::NotFound(); @@ -1401,8 +1498,12 @@ Status Redis::ZRemrangebylex(const Slice& key, const Slice& min, const Slice& ma std::string meta_value; BaseMetaKey base_meta_key(key); - Status s = db_->Get(read_options, handles_[kZsetsMetaCF], base_meta_key.Encode(), &meta_value); + Status s = db_->Get(read_options, handles_[kMetaCF], base_meta_key.Encode(), &meta_value); if (s.ok()) { + auto type = static_cast(static_cast(meta_value[0])); + if (type != Type::kZset) { + return Status::InvalidArgument("WRONGTYPE Operation against a key holding the wrong kind of value"); + } ParsedZSetsMetaValue parsed_zsets_meta_value(&meta_value); if (parsed_zsets_meta_value.IsStale() || parsed_zsets_meta_value.Count() == 0) { return Status::NotFound(); @@ -1447,7 +1548,7 @@ Status Redis::ZRemrangebylex(const Slice& key, const Slice& min, const Slice& ma return Status::InvalidArgument("zset size overflow"); } parsed_zsets_meta_value.ModifyCount(-del_cnt); - batch.Put(handles_[kZsetsMetaCF], base_meta_key.Encode(), meta_value); + batch.Put(handles_[kMetaCF], base_meta_key.Encode(), meta_value); *ret = del_cnt; } } else { @@ -1463,8 +1564,12 @@ Status Redis::ZsetsExpire(const Slice& key, uint64_t ttl) { ScopeRecordLock l(lock_mgr_, key); BaseMetaKey base_meta_key(key); - Status s = db_->Get(default_read_options_, handles_[kZsetsMetaCF], base_meta_key.Encode(), &meta_value); + Status s = db_->Get(default_read_options_, handles_[kMetaCF], base_meta_key.Encode(), &meta_value); if (s.ok()) { + auto type = static_cast(static_cast(meta_value[0])); + if (type != Type::kZset) { + return Status::InvalidArgument("WRONGTYPE Operation against a key holding the wrong kind of value"); + } ParsedZSetsMetaValue parsed_zsets_meta_value(&meta_value); if (parsed_zsets_meta_value.IsStale()) { return Status::NotFound("Stale"); @@ -1477,7 +1582,7 @@ Status Redis::ZsetsExpire(const Slice& key, uint64_t ttl) { } else { parsed_zsets_meta_value.InitialMetaValue(); } - s = db_->Put(default_write_options_, handles_[kZsetsMetaCF], base_meta_key.Encode(), meta_value); + s = db_->Put(default_write_options_, handles_[kMetaCF], base_meta_key.Encode(), meta_value); } return s; } @@ -1487,9 +1592,13 @@ Status Redis::ZsetsDel(const Slice& key) { ScopeRecordLock l(lock_mgr_, key); BaseMetaKey base_meta_key(key); - Status s = db_->Get(default_read_options_, handles_[kZsetsMetaCF], base_meta_key.Encode(), &meta_value); + Status s = db_->Get(default_read_options_, handles_[kMetaCF], base_meta_key.Encode(), &meta_value); if (s.ok()) { ParsedZSetsMetaValue parsed_zsets_meta_value(&meta_value); + if (!parsed_zsets_meta_value.IsSameType(Type::kZset)) { + std::cout << "error" << std::endl; + return Status::InvalidArgument("WRONGTYPE Operation against a key holding the wrong kind of value"); + } if (parsed_zsets_meta_value.IsStale()) { return Status::NotFound("Stale"); } else if (parsed_zsets_meta_value.Count() == 0) { @@ -1497,7 +1606,7 @@ Status Redis::ZsetsDel(const Slice& key) { } else { uint32_t statistic = parsed_zsets_meta_value.Count(); parsed_zsets_meta_value.InitialMetaValue(); - s = db_->Put(default_write_options_, handles_[kZsetsMetaCF], base_meta_key.Encode(), meta_value); + s = db_->Put(default_write_options_, handles_[kMetaCF], base_meta_key.Encode(), meta_value); UpdateSpecificKeyStatistics(DataType::kZSets, key.ToString(), statistic); } } @@ -1509,8 +1618,12 @@ Status Redis::ZsetsExpireat(const Slice& key, uint64_t timestamp) { ScopeRecordLock l(lock_mgr_, key); BaseMetaKey base_meta_key(key); - Status s = db_->Get(default_read_options_, handles_[kZsetsMetaCF], base_meta_key.Encode(), &meta_value); + Status s = db_->Get(default_read_options_, handles_[kMetaCF], base_meta_key.Encode(), &meta_value); if (s.ok()) { + auto type = static_cast(static_cast(meta_value[0])); + if (type != Type::kZset) { + return Status::InvalidArgument("WRONGTYPE Operation against a key holding the wrong kind of value"); + } ParsedZSetsMetaValue parsed_zsets_meta_value(&meta_value); if (parsed_zsets_meta_value.IsStale()) { return Status::NotFound("Stale"); @@ -1522,7 +1635,7 @@ Status Redis::ZsetsExpireat(const Slice& key, uint64_t timestamp) { } else { parsed_zsets_meta_value.InitialMetaValue(); } - return db_->Put(default_write_options_, handles_[kZsetsMetaCF], base_meta_key.Encode(), meta_value); + return db_->Put(default_write_options_, handles_[kMetaCF], base_meta_key.Encode(), meta_value); } } return s; @@ -1547,8 +1660,12 @@ Status Redis::ZScan(const Slice& key, int64_t cursor, const std::string& pattern read_options.snapshot = snapshot; BaseMetaKey base_meta_key(key); - Status s = db_->Get(read_options, handles_[kZsetsMetaCF], base_meta_key.Encode(), &meta_value); + Status s = db_->Get(read_options, handles_[kMetaCF], base_meta_key.Encode(), &meta_value); if (s.ok()) { + auto type = static_cast(static_cast(meta_value[0])); + if (type != Type::kZset) { + return Status::InvalidArgument("WRONGTYPE Operation against a key holding the wrong kind of value"); + } ParsedZSetsMetaValue parsed_zsets_meta_value(&meta_value); if (parsed_zsets_meta_value.IsStale() || parsed_zsets_meta_value.Count() == 0) { *next_cursor = 0; @@ -1609,8 +1726,12 @@ Status Redis::ZsetsPersist(const Slice& key) { ScopeRecordLock l(lock_mgr_, key); BaseMetaKey base_meta_key(key); - Status s = db_->Get(default_read_options_, handles_[kZsetsMetaCF], base_meta_key.Encode(), &meta_value); + Status s = db_->Get(default_read_options_, handles_[kMetaCF], base_meta_key.Encode(), &meta_value); if (s.ok()) { + auto type = static_cast(static_cast(meta_value[0])); + if (type != Type::kZset) { + return Status::InvalidArgument("WRONGTYPE Operation against a key holding the wrong kind of value"); + } ParsedZSetsMetaValue parsed_zsets_meta_value(&meta_value); if (parsed_zsets_meta_value.IsStale()) { return Status::NotFound("Stale"); @@ -1622,7 +1743,7 @@ Status Redis::ZsetsPersist(const Slice& key) { return Status::NotFound("Not have an associated timeout"); } else { parsed_zsets_meta_value.SetEtime(0); - return db_->Put(default_write_options_, handles_[kZsetsMetaCF], base_meta_key.Encode(), meta_value); + return db_->Put(default_write_options_, handles_[kMetaCF], base_meta_key.Encode(), meta_value); } } } @@ -1633,8 +1754,12 @@ Status Redis::ZsetsTTL(const Slice& key, uint64_t* timestamp) { std::string meta_value; BaseMetaKey base_meta_key(key); - Status s = db_->Get(default_read_options_, handles_[kZsetsMetaCF], base_meta_key.Encode(), &meta_value); + Status s = db_->Get(default_read_options_, handles_[kMetaCF], base_meta_key.Encode(), &meta_value); if (s.ok()) { + auto type = static_cast(static_cast(meta_value[0])); + if (type != Type::kZset) { + return Status::InvalidArgument("WRONGTYPE Operation against a key holding the wrong kind of value"); + } ParsedZSetsMetaValue parsed_zsets_meta_value(&meta_value); if (parsed_zsets_meta_value.IsStale()) { *timestamp = -2; @@ -1666,7 +1791,7 @@ Status Redis::ZsetsRename(const Slice& key, Redis* new_inst, const Slice& newkey BaseMetaKey base_meta_key(key); BaseMetaKey base_meta_newkey(newkey); - Status s = db_->Get(default_read_options_, handles_[kZsetsMetaCF], base_meta_key.Encode(), &meta_value); + Status s = db_->Get(default_read_options_, handles_[kMetaCF], base_meta_key.Encode(), &meta_value); if (s.ok()) { ParsedZSetsMetaValue parsed_zsets_meta_value(&meta_value); if (parsed_zsets_meta_value.IsStale()) { @@ -1676,12 +1801,12 @@ Status Redis::ZsetsRename(const Slice& key, Redis* new_inst, const Slice& newkey } // copy a new zset with newkey statistic = parsed_zsets_meta_value.Count(); - s = new_inst->GetDB()->Put(default_write_options_, handles_[kZsetsMetaCF], base_meta_newkey.Encode(), meta_value); + s = new_inst->GetDB()->Put(default_write_options_, handles_[kMetaCF], base_meta_newkey.Encode(), meta_value); new_inst->UpdateSpecificKeyStatistics(DataType::kZSets, newkey.ToString(), statistic); // ZsetsDel key parsed_zsets_meta_value.InitialMetaValue(); - s = db_->Put(default_write_options_, handles_[kZsetsMetaCF], base_meta_key.Encode(), meta_value); + s = db_->Put(default_write_options_, handles_[kMetaCF], base_meta_key.Encode(), meta_value); UpdateSpecificKeyStatistics(DataType::kZSets, key.ToString(), statistic); } return s; @@ -1695,7 +1820,7 @@ Status Redis::ZsetsRenamenx(const Slice& key, Redis* new_inst, const Slice& newk BaseMetaKey base_meta_key(key); BaseMetaKey base_meta_newkey(newkey); - Status s = db_->Get(default_read_options_, handles_[kZsetsMetaCF], base_meta_key.Encode(), &meta_value); + Status s = db_->Get(default_read_options_, handles_[kMetaCF], base_meta_key.Encode(), &meta_value); if (s.ok()) { ParsedZSetsMetaValue parsed_zsets_meta_value(&meta_value); if (parsed_zsets_meta_value.IsStale()) { @@ -1705,7 +1830,7 @@ Status Redis::ZsetsRenamenx(const Slice& key, Redis* new_inst, const Slice& newk } // check if newkey exist. std::string new_meta_value; - s = new_inst->GetDB()->Get(default_read_options_, handles_[kZsetsMetaCF], base_meta_newkey.Encode(), + s = new_inst->GetDB()->Get(default_read_options_, handles_[kMetaCF], base_meta_newkey.Encode(), &new_meta_value); if (s.ok()) { ParsedSetsMetaValue parsed_zsets_new_meta_value(&new_meta_value); @@ -1716,12 +1841,12 @@ Status Redis::ZsetsRenamenx(const Slice& key, Redis* new_inst, const Slice& newk // copy a new zset with newkey statistic = parsed_zsets_meta_value.Count(); - s = new_inst->GetDB()->Put(default_write_options_, handles_[kZsetsMetaCF], base_meta_newkey.Encode(), meta_value); + s = new_inst->GetDB()->Put(default_write_options_, handles_[kMetaCF], base_meta_newkey.Encode(), meta_value); new_inst->UpdateSpecificKeyStatistics(DataType::kZSets, newkey.ToString(), statistic); // ZsetsDel key parsed_zsets_meta_value.InitialMetaValue(); - s = db_->Put(default_write_options_, handles_[kZsetsMetaCF], base_meta_key.Encode(), meta_value); + s = db_->Put(default_write_options_, handles_[kMetaCF], base_meta_key.Encode(), meta_value); UpdateSpecificKeyStatistics(DataType::kZSets, key.ToString(), statistic); } return s; @@ -1736,10 +1861,13 @@ void Redis::ScanZsets() { auto current_time = static_cast(time(nullptr)); INFO("***************rocksdb instance: {} ZSets Meta Data***************", index_); - auto meta_iter = db_->NewIterator(iterator_options, handles_[kZsetsMetaCF]); + auto meta_iter = db_->NewIterator(iterator_options, handles_[kMetaCF]); for (meta_iter->SeekToFirst(); meta_iter->Valid(); meta_iter->Next()) { ParsedBaseMetaKey parsed_meta_key(meta_iter->key()); ParsedZSetsMetaValue parsed_zsets_meta_value(meta_iter->value()); + if (!parsed_zsets_meta_value.IsSameType(Type::kZset)) { + continue; + } int32_t survival_time = 0; if (parsed_zsets_meta_value.Etime() != 0) { survival_time = diff --git a/src/storage/src/storage.cc b/src/storage/src/storage.cc index 776ca09a7..5385c1fea 100644 --- a/src/storage/src/storage.cc +++ b/src/storage/src/storage.cc @@ -1196,109 +1196,28 @@ Status Storage::ZScan(const Slice& key, int64_t cursor, const std::string& patte int32_t Storage::Expire(const Slice& key, uint64_t ttl) { int32_t ret = 0; - bool is_corruption = false; - auto& inst = GetDBInstance(key); // Strings - Status s = inst->StringsExpire(key, ttl); - if (s.ok()) { - ret++; - } else if (!s.IsNotFound()) { - is_corruption = true; - } - - // Hash - s = inst->HashesExpire(key, ttl); + Status s = inst->Expire(key, ttl); if (s.ok()) { ret++; } else if (!s.IsNotFound()) { - is_corruption = true; - } - - // Sets - s = inst->SetsExpire(key, ttl); - if (s.ok()) { - ret++; - } else if (!s.IsNotFound()) { - is_corruption = true; - } - - // Lists - s = inst->ListsExpire(key, ttl); - if (s.ok()) { - ret++; - } else if (!s.IsNotFound()) { - is_corruption = true; - } - - // Zsets - s = inst->ZsetsExpire(key, ttl); - if (s.ok()) { - ret++; - } else if (!s.IsNotFound()) { - is_corruption = true; - } - - if (is_corruption) { return -1; - } else { - return ret; } + return ret; } int64_t Storage::Del(const std::vector& keys) { Status s; int64_t count = 0; - bool is_corruption = false; - for (const auto& key : keys) { auto& inst = GetDBInstance(key); - // Strings - Status s = inst->StringsDel(key); - if (s.ok()) { - count++; - } else if (!s.IsNotFound()) { - is_corruption = true; - } - - // Hashes - s = inst->HashesDel(key); + s = inst->Del(key); if (s.ok()) { count++; - } else if (!s.IsNotFound()) { - is_corruption = true; - } - - // Sets - s = inst->SetsDel(key); - if (s.ok()) { - count++; - } else if (!s.IsNotFound()) { - is_corruption = true; - } - - // Lists - s = inst->ListsDel(key); - if (s.ok()) { - count++; - } else if (!s.IsNotFound()) { - is_corruption = true; - } - - // ZSets - s = inst->ZsetsDel(key); - if (s.ok()) { - count++; - } else if (!s.IsNotFound()) { - is_corruption = true; } } - - if (is_corruption) { - return -1; - } else { - return count; - } + return count; } int64_t Storage::DelByType(const std::vector& keys, const DataType& type) { @@ -1374,55 +1293,19 @@ int64_t Storage::DelByType(const std::vector& keys, const DataType& int64_t Storage::Exists(const std::vector& keys) { int64_t count = 0; - int32_t ret; - uint64_t llen; - std::string value; Status s; bool is_corruption = false; for (const auto& key : keys) { auto& inst = GetDBInstance(key); - s = inst->Get(key, &value); - if (s.ok()) { - count++; - } else if (!s.IsNotFound()) { - is_corruption = true; - } - - s = inst->HLen(key, &ret); - if (s.ok()) { - count++; - } else if (!s.IsNotFound()) { - is_corruption = true; - } - - s = inst->SCard(key, &ret); - if (s.ok()) { - count++; - } else if (!s.IsNotFound()) { - is_corruption = true; - } - - s = inst->LLen(key, &llen); - if (s.ok()) { - count++; - } else if (!s.IsNotFound()) { - is_corruption = true; - } - - s = inst->ZCard(key, &ret); + s = inst->Exists(key); if (s.ok()) { count++; } else if (!s.IsNotFound()) { - is_corruption = true; + return -1; } } - - if (is_corruption) { - return -1; - } else { - return count; - } + return count; } int64_t Storage::Scan(const DataType& dtype, int64_t cursor, const std::string& pattern, int64_t count, @@ -1679,225 +1562,48 @@ Status Storage::Scanx(const DataType& data_type, const std::string& start_key, c int32_t Storage::Expireat(const Slice& key, uint64_t timestamp) { Status s; int32_t count = 0; - bool is_corruption = false; - auto& inst = GetDBInstance(key); s = inst->StringsExpireat(key, timestamp); if (s.ok()) { count++; } else if (!s.IsNotFound()) { - is_corruption = true; - } - - s = inst->HashesExpireat(key, timestamp); - if (s.ok()) { - count++; - } else if (!s.IsNotFound()) { - is_corruption = true; - } - - s = inst->SetsExpireat(key, timestamp); - if (s.ok()) { - count++; - } else if (!s.IsNotFound()) { - is_corruption = true; - } - - s = inst->ListsExpireat(key, timestamp); - if (s.ok()) { - count++; - } else if (!s.IsNotFound()) { - is_corruption = true; - } - - s = inst->ZsetsExpireat(key, timestamp); - if (s.ok()) { - count++; - } else if (!s.IsNotFound()) { - is_corruption = true; - } - - if (is_corruption) { return -1; } return count; } -int32_t Storage::Persist(const Slice& key, std::map* type_status) { - Status s; - int32_t count = 0; - bool is_corruption = false; - +int32_t Storage::Persist(const Slice& key) { auto& inst = GetDBInstance(key); - s = inst->StringsPersist(key); + Status s = inst->Persist(key); if (s.ok()) { - count++; - } else if (!s.IsNotFound()) { - is_corruption = true; - (*type_status)[DataType::kStrings] = s; - } - - s = inst->HashesPersist(key); - if (s.ok()) { - count++; - } else if (!s.IsNotFound()) { - is_corruption = true; - (*type_status)[DataType::kHashes] = s; - } - - s = inst->SetsPersist(key); - if (s.ok()) { - count++; - } else if (!s.IsNotFound()) { - is_corruption = true; - (*type_status)[DataType::kSets] = s; - } - - s = inst->ListsPersist(key); - if (s.ok()) { - count++; - } else if (!s.IsNotFound()) { - is_corruption = true; - (*type_status)[DataType::kLists] = s; - } - - s = inst->ZsetsPersist(key); - if (s.ok()) { - count++; - } else if (!s.IsNotFound()) { - is_corruption = true; - (*type_status)[DataType::kZSets] = s; - } - - if (is_corruption) { - return -1; + return 1; } else { - return count; + return 0; } } -std::map Storage::TTL(const Slice& key, std::map* type_status) { - Status s; - std::map ret; +int32_t Storage::TTL(const Slice& key) { uint64_t timestamp = 0; auto& inst = GetDBInstance(key); - s = inst->StringsTTL(key, ×tamp); - if (s.ok() || s.IsNotFound()) { - ret[DataType::kStrings] = timestamp; - } else if (!s.IsNotFound()) { - ret[DataType::kStrings] = -3; - (*type_status)[DataType::kStrings] = s; - } - - s = inst->HashesTTL(key, ×tamp); - if (s.ok() || s.IsNotFound()) { - ret[DataType::kHashes] = timestamp; - } else if (!s.IsNotFound()) { - ret[DataType::kHashes] = -3; - (*type_status)[DataType::kHashes] = s; - } - - s = inst->ListsTTL(key, ×tamp); - if (s.ok() || s.IsNotFound()) { - ret[DataType::kLists] = timestamp; - } else if (!s.IsNotFound()) { - ret[DataType::kLists] = -3; - (*type_status)[DataType::kLists] = s; - } - - s = inst->SetsTTL(key, ×tamp); - if (s.ok() || s.IsNotFound()) { - ret[DataType::kSets] = timestamp; - } else if (!s.IsNotFound()) { - ret[DataType::kSets] = -3; - (*type_status)[DataType::kSets] = s; - } - - s = inst->ZsetsTTL(key, ×tamp); - if (s.ok() || s.IsNotFound()) { - ret[DataType::kZSets] = timestamp; - } else if (!s.IsNotFound()) { - ret[DataType::kZSets] = -3; - (*type_status)[DataType::kZSets] = s; + Status s = inst->TTL(key, ×tamp); + if (s.ok()) { + return timestamp; + } else { + return -2; } - return ret; } -Status Storage::GetType(const std::string& key, bool single, std::vector& types) { - types.clear(); - - Status s; - std::string value; +Status Storage::GetType(const std::string& key, std::string& types) { auto& inst = GetDBInstance(key); - s = inst->Get(key, &value); - if (s.ok()) { - types.emplace_back("string"); - } else if (!s.IsNotFound()) { - return s; - } - if (single && !types.empty()) { - return s; - } - - int32_t hashes_len = 0; - s = inst->HLen(key, &hashes_len); - if (s.ok() && hashes_len != 0) { - types.emplace_back("hash"); - } else if (!s.IsNotFound()) { - return s; - } - if (single && !types.empty()) { - return s; - } - - uint64_t lists_len = 0; - s = inst->LLen(key, &lists_len); - if (s.ok() && lists_len != 0) { - types.emplace_back("list"); - } else if (!s.IsNotFound()) { - return s; - } - if (single && !types.empty()) { - return s; - } - - int32_t zsets_size = 0; - s = inst->ZCard(key, &zsets_size); - if (s.ok() && zsets_size != 0) { - types.emplace_back("zset"); - } else if (!s.IsNotFound()) { - return s; - } - if (single && !types.empty()) { - return s; - } - - int32_t sets_size = 0; - s = inst->SCard(key, &sets_size); - if (s.ok() && sets_size != 0) { - types.emplace_back("set"); - } else if (!s.IsNotFound()) { - return s; - } - if (single && types.empty()) { - types.emplace_back("none"); - } - return Status::OK(); + Status s = inst->GetType(key, types); + return s; } Status Storage::Keys(const DataType& data_type, const std::string& pattern, std::vector* keys) { keys->clear(); std::vector types; - if (data_type == DataType::kAll) { - types.push_back(DataType::kStrings); - types.push_back(DataType::kHashes); - types.push_back(DataType::kLists); - types.push_back(DataType::kZSets); - types.push_back(DataType::kSets); - } else { - types.push_back(data_type); - } + types.push_back(data_type); for (const auto& type : types) { std::vector inst_iters; @@ -2419,34 +2125,9 @@ void Storage::GetRocksDBInfo(std::string& info) { } int64_t Storage::IsExist(const Slice& key, std::map* type_status) { - std::string value; - int32_t ret = 0; int64_t type_count = 0; auto& inst = GetDBInstance(key); - Status s = inst->Get(key, &value); - (*type_status)[DataType::kStrings] = s; - if (s.ok()) { - type_count++; - } - s = inst->HLen(key, &ret); - (*type_status)[DataType::kHashes] = s; - if (s.ok()) { - type_count++; - } - s = inst->SCard(key, &ret); - (*type_status)[DataType::kSets] = s; - if (s.ok()) { - type_count++; - } - uint64_t llen = 0; - s = inst->LLen(key, &llen); - (*type_status)[DataType::kLists] = s; - if (s.ok()) { - type_count++; - } - - s = inst->ZCard(key, &ret); - (*type_status)[DataType::kZSets] = s; + Status s = inst->IsExist(key); if (s.ok()) { type_count++; } diff --git a/src/storage/src/strings_filter.h b/src/storage/src/strings_filter.h deleted file mode 100644 index 245acf4e5..000000000 --- a/src/storage/src/strings_filter.h +++ /dev/null @@ -1,54 +0,0 @@ -// Copyright (c) 2017-present, Qihoo, Inc. All rights reserved. -// This source code is licensed under the BSD-style license found in the -// LICENSE file in the root directory of this source tree. An additional grant -// of patent rights can be found in the PATENTS file in the same directory. - -#ifndef SRC_STRINGS_FILTER_H_ -#define SRC_STRINGS_FILTER_H_ - -#include -#include - -#include "rocksdb/compaction_filter.h" -#include "src/debug.h" -#include "src/strings_value_format.h" - -namespace storage { - -class StringsFilter : public rocksdb::CompactionFilter { - public: - StringsFilter() = default; - bool Filter(int level, const rocksdb::Slice& key, const rocksdb::Slice& value, std::string* new_value, - bool* value_changed) const override { - int64_t unix_time; - rocksdb::Env::Default()->GetCurrentTime(&unix_time); - auto cur_time = static_cast(unix_time); - ParsedStringsValue parsed_strings_value(value); - TRACE("==========================START=========================="); - TRACE("[StringsFilter], key: %s, value = %s, timestamp: %llu, cur_time: %d", key.ToString().c_str(), - parsed_strings_value.UserValue().ToString().c_str(), parsed_strings_value.Etime(), cur_time); - - if (parsed_strings_value.Etime() != 0 && parsed_strings_value.Etime() < cur_time) { - TRACE("Drop[Stale]"); - return true; - } else { - TRACE("Reserve"); - return false; - } - } - - const char* Name() const override { return "StringsFilter"; } -}; - -class StringsFilterFactory : public rocksdb::CompactionFilterFactory { - public: - StringsFilterFactory() = default; - std::unique_ptr CreateCompactionFilter( - const rocksdb::CompactionFilter::Context& context) override { - return std::unique_ptr(new StringsFilter()); - } - const char* Name() const override { return "StringsFilterFactory"; } -}; - -} // namespace storage -#endif // SRC_STRINGS_FILTER_H_ diff --git a/src/storage/src/strings_value_format.h b/src/storage/src/strings_value_format.h index 17be96398..153164c67 100644 --- a/src/storage/src/strings_value_format.h +++ b/src/storage/src/strings_value_format.h @@ -13,18 +13,18 @@ namespace storage { /* - * | value | reserve | cdate | timestamp | - * | | 16B | 8B | 8B | + * | type | value | reserve | cdate | timestamp | + * | 1B | | 16B | 8B | 8B | */ class StringsValue : public InternalValue { public: - explicit StringsValue(const rocksdb::Slice& user_value) : InternalValue(user_value) {} + explicit StringsValue(const rocksdb::Slice& user_value) : InternalValue(Type::kString, user_value) {} virtual rocksdb::Slice Encode() override { size_t usize = user_value_.size(); - size_t needed = usize + kSuffixReserveLength + 2 * kTimestampLength; + size_t needed = kTypeLength + usize + kSuffixReserveLength + 2 * kTimestampLength; char* dst = ReAllocIfNeeded(needed); - char* start_pos = dst; - + memcpy(dst, &type_, sizeof(type_)); + dst += sizeof(type_); memcpy(dst, user_value_.data(), usize); dst += usize; memcpy(dst, reserve_, kSuffixReserveLength); @@ -32,7 +32,7 @@ class StringsValue : public InternalValue { EncodeFixed64(dst, ctime_); dst += kTimestampLength; EncodeFixed64(dst, etime_); - return rocksdb::Slice(start_pos, needed); + return {start_, needed}; } }; @@ -40,28 +40,41 @@ class ParsedStringsValue : public ParsedInternalValue { public: // Use this constructor after rocksdb::DB::Get(); explicit ParsedStringsValue(std::string* internal_value_str) : ParsedInternalValue(internal_value_str) { - if (internal_value_str->size() >= kStringsValueSuffixLength) { - user_value_ = rocksdb::Slice(internal_value_str->data(), internal_value_str->size() - kStringsValueSuffixLength); - memcpy(reserve_, internal_value_str->data() + user_value_.size(), kSuffixReserveLength); - ctime_ = DecodeFixed64(internal_value_str->data() + user_value_.size() + kSuffixReserveLength); - etime_ = DecodeFixed64(internal_value_str->data() + user_value_.size() + kSuffixReserveLength + kTimestampLength); + if (internal_value_str->size() >= kStringsValueMinLength) { + size_t offset = 0; + type_ = static_cast(static_cast((*internal_value_str)[0])); + offset += kTypeLength; + user_value_ = rocksdb::Slice(internal_value_str->data() + offset, + internal_value_str->size() - kStringsValueSuffixLength - offset); + offset += user_value_.size(); + memcpy(reserve_, internal_value_str->data() + offset, kSuffixReserveLength); + offset += kSuffixReserveLength; + ctime_ = DecodeFixed64(internal_value_str->data() + offset); + offset += sizeof(ctime_); + etime_ = DecodeFixed64(internal_value_str->data() + offset); } } // Use this constructor in rocksdb::CompactionFilter::Filter(); explicit ParsedStringsValue(const rocksdb::Slice& internal_value_slice) : ParsedInternalValue(internal_value_slice) { - if (internal_value_slice.size() >= kStringsValueSuffixLength) { + if (internal_value_slice.size() >= kStringsValueMinLength) { + size_t offset = 0; + type_ = static_cast(static_cast(internal_value_slice[0])); + offset += kTypeLength; user_value_ = - rocksdb::Slice(internal_value_slice.data(), internal_value_slice.size() - kStringsValueSuffixLength); - memcpy(reserve_, internal_value_slice.data() + user_value_.size(), kSuffixReserveLength); - ctime_ = DecodeFixed64(internal_value_slice.data() + user_value_.size() + kSuffixReserveLength); - etime_ = - DecodeFixed64(internal_value_slice.data() + user_value_.size() + kSuffixReserveLength + kTimestampLength); + rocksdb::Slice(internal_value_slice.data(), internal_value_slice.size() - kStringsValueSuffixLength - offset); + offset += user_value_.size(); + memcpy(reserve_, internal_value_slice.data() + offset, kSuffixReserveLength); + offset += kSuffixReserveLength; + ctime_ = DecodeFixed64(internal_value_slice.data() + offset); + offset += kTimestampLength; + etime_ = DecodeFixed64(internal_value_slice.data() + offset); } } void StripSuffix() override { if (value_) { + value_->erase(0, kTypeLength); value_->erase(value_->size() - kStringsValueSuffixLength, kStringsValueSuffixLength); } } @@ -85,7 +98,8 @@ class ParsedStringsValue : public ParsedInternalValue { } private: - const size_t kStringsValueSuffixLength = 2 * kTimestampLength + kSuffixReserveLength; + const static size_t kStringsValueSuffixLength = 2 * kTimestampLength + kSuffixReserveLength; + const static size_t kStringsValueMinLength = kStringsValueSuffixLength + kTypeLength; }; } // namespace storage diff --git a/src/storage/src/type_iterator.h b/src/storage/src/type_iterator.h index 3c17a8e37..68a528946 100644 --- a/src/storage/src/type_iterator.h +++ b/src/storage/src/type_iterator.h @@ -18,7 +18,6 @@ #include "src/base_data_key_format.h" #include "src/base_key_format.h" #include "src/base_meta_value_format.h" -#include "src/debug.h" #include "src/lists_meta_value_format.h" #include "src/mutex.h" #include "src/strings_value_format.h" @@ -107,6 +106,10 @@ class StringsIterator : public TypeIterator { ~StringsIterator() {} bool ShouldSkip() override { + auto type = static_cast(static_cast(raw_iter_->value()[0])); + if (type == Type::kString) { + return true; + } ParsedStringsValue parsed_value(raw_iter_->value()); if (parsed_value.IsStale()) { return true; @@ -134,6 +137,10 @@ class HashesIterator : public TypeIterator { ~HashesIterator() {} bool ShouldSkip() override { + auto type = static_cast(static_cast(raw_iter_->value()[0])); + if (type == Type::kHash) { + return true; + } ParsedHashesMetaValue parsed_meta_value(raw_iter_->value()); if (parsed_meta_value.IsStale() || parsed_meta_value.Count() == 0) { return true; @@ -160,6 +167,10 @@ class ListsIterator : public TypeIterator { ~ListsIterator() {} bool ShouldSkip() override { + auto type = static_cast(static_cast(raw_iter_->value()[0])); + if (type == Type::kList) { + return true; + } ParsedListsMetaValue parsed_meta_value(raw_iter_->value()); if (parsed_meta_value.IsStale() || parsed_meta_value.Count() == 0) { return true; @@ -186,6 +197,10 @@ class SetsIterator : public TypeIterator { ~SetsIterator() {} bool ShouldSkip() override { + auto type = static_cast(static_cast(raw_iter_->value()[0])); + if (type == Type::kSet) { + return true; + } ParsedSetsMetaValue parsed_meta_value(raw_iter_->value()); if (parsed_meta_value.IsStale() || parsed_meta_value.Count() == 0) { return true; @@ -212,6 +227,10 @@ class ZsetsIterator : public TypeIterator { ~ZsetsIterator() {} bool ShouldSkip() override { + auto type = static_cast(static_cast(raw_iter_->value()[0])); + if (type == Type::kZset) { + return true; + } ParsedZSetsMetaValue parsed_meta_value(raw_iter_->value()); if (parsed_meta_value.IsStale() || parsed_meta_value.Count() == 0) { return true; @@ -230,6 +249,49 @@ class ZsetsIterator : public TypeIterator { std::string pattern_; }; +class AllIterator : public TypeIterator { + public: + AllIterator(const rocksdb::ReadOptions& options, rocksdb::DB* db, ColumnFamilyHandle* handle, + const std::string& pattern) + : TypeIterator(options, db, handle), pattern_(pattern) {} + ~AllIterator() {} + + bool ShouldSkip() override { + std::string user_value; + auto type = static_cast(static_cast(raw_iter_->value()[0])); + if (type == Type::kZset || type == Type::kSet || type == Type::kHash) { + ParsedBaseMetaValue parsed_meta_value(raw_iter_->value()); + user_value = parsed_meta_value.UserValue().ToString(); + if (parsed_meta_value.IsStale() || parsed_meta_value.Count() == 0) { + return true; + } + } else if (type == Type::kList) { + ParsedListsMetaValue parsed_meta_value(raw_iter_->value()); + user_value = parsed_meta_value.UserValue().ToString(); + if (parsed_meta_value.IsStale() || parsed_meta_value.Count() == 0) { + return true; + } + } else { + ParsedStringsValue parsed_value(raw_iter_->value()); + user_value = parsed_value.UserValue().ToString(); + if (parsed_value.IsStale()) { + return true; + } + } + + ParsedBaseMetaKey parsed_key(raw_iter_->key().ToString()); + if (StringMatch(pattern_.data(), pattern_.size(), parsed_key.Key().data(), parsed_key.Key().size(), 0) == 0) { + return true; + } + user_key_ = parsed_key.Key().ToString(); + user_value_ = user_value; + return false; + } + + private: + std::string pattern_; +}; + using IterSptr = std::shared_ptr; class MinMergeComparator { diff --git a/src/storage/src/zsets_filter.h b/src/storage/src/zsets_filter.h index b141c41fd..ca90c924b 100644 --- a/src/storage/src/zsets_filter.h +++ b/src/storage/src/zsets_filter.h @@ -30,8 +30,8 @@ class ZSetsScoreFilter : public rocksdb::CompactionFilter { UNUSED(new_value); UNUSED(value_changed); ParsedZSetsScoreKey parsed_zsets_score_key(key); - TRACE("==========================START=========================="); - TRACE("[ScoreFilter], key: %s, score = %lf, member = %s, version = %lld", + DEBUG("==========================START=========================="); + DEBUG("[ScoreFilter], key: %s, score = %lf, member = %s, version = %lld", parsed_zsets_score_key.key().ToString().c_str(), parsed_zsets_score_key.score(), parsed_zsets_score_key.member().ToString().c_str(), parsed_zsets_score_key.Version()); @@ -58,27 +58,27 @@ class ZSetsScoreFilter : public rocksdb::CompactionFilter { meta_not_found_ = true; } else { cur_key_ = ""; - TRACE("Reserve[Get meta_key faild]"); + DEBUG("Reserve[Get meta_key faild]"); return false; } } if (meta_not_found_) { - TRACE("Drop[Meta key not exist]"); + DEBUG("Drop[Meta key not exist]"); return true; } int64_t unix_time; rocksdb::Env::Default()->GetCurrentTime(&unix_time); if (cur_meta_etime_ != 0 && cur_meta_etime_ < static_cast(unix_time)) { - TRACE("Drop[Timeout]"); + DEBUG("Drop[Timeout]"); return true; } if (cur_meta_version_ > parsed_zsets_score_key.Version()) { - TRACE("Drop[score_key_version < cur_meta_version]"); + DEBUG("Drop[score_key_version < cur_meta_version]"); return true; } else { - TRACE("Reserve[score_key_version == cur_meta_version]"); + DEBUG("Reserve[score_key_version == cur_meta_version]"); return false; } } diff --git a/src/storage/tests/flush_oldest_cf_test.cc b/src/storage/tests/flush_oldest_cf_test.cc deleted file mode 100644 index 2558dfc11..000000000 --- a/src/storage/tests/flush_oldest_cf_test.cc +++ /dev/null @@ -1,484 +0,0 @@ -/* - * Copyright (c) 2024-present, Qihoo, Inc. All rights reserved. - * This source code is licensed under the BSD-style license found in the - * LICENSE file in the root directory of this source tree. An additional grant - * of patent rights can be found in the PATENTS file in the same directory. - */ - -#include "gtest/gtest.h" - -#include -#include -#include -#include -#include -#include - -#include "fmt/core.h" -#include "gtest/gtest.h" -#include "rocksdb/db.h" -#include "rocksdb/listener.h" -#include "rocksdb/metadata.h" -#include "rocksdb/options.h" - -#include "pstd/log.h" -#include "pstd/thread_pool.h" -#include "src/log_index.h" -#include "src/redis.h" -#include "storage/storage.h" -#include "storage/util.h" - -class LogIniter { - public: - LogIniter() { - logger::Init("./flush_oldest_cf_test.log"); - spdlog::set_level(spdlog::level::info); - } -}; - -LogIniter log_initer; - -using LogIndex = int64_t; - -class LogQueue : public pstd::noncopyable { - public: - using WriteCallback = std::function; - - explicit LogQueue(WriteCallback&& cb) : write_cb_(std::move(cb)) { consumer_.SetMaxIdleThread(1); } - - void AppendLog(const pikiwidb::Binlog& log, std::promise&& promise) { - auto task = [&] { - auto idx = next_log_idx_.fetch_add(1); - auto s = write_cb_(log, idx); - promise.set_value(s); - }; - consumer_.ExecuteTask(std::move(task)); - } - - private: - WriteCallback write_cb_ = nullptr; - pstd::ThreadPool consumer_; - std::atomic next_log_idx_{1}; -}; - -class FlushOldestCFTest : public ::testing::Test { - public: - FlushOldestCFTest() - : log_queue_([this](const pikiwidb::Binlog& log, LogIndex log_idx) { return db_.OnBinlogWrite(log, log_idx); }) { - options_.options.create_if_missing = true; - options_.options.max_background_jobs = 10; - options_.db_instance_num = 1; - options_.raft_timeout_s = 9000000; - options_.append_log_function = [this](const pikiwidb::Binlog& log, std::promise&& promise) { - log_queue_.AppendLog(log, std::move(promise)); - }; - options_.do_snapshot_function = [](int64_t log_index, bool sync) {}; - options_.max_gap = 15; - write_options_.disableWAL = true; - } - - ~FlushOldestCFTest() { rocksdb::DestroyDB(db_path_, rocksdb::Options()); } - - void SetUp() override { - if (access(db_path_.c_str(), F_OK) == 0) { - std::filesystem::remove_all(db_path_.c_str()); - } - mkdir(db_path_.c_str(), 0755); - auto s = db_.Open(options_, db_path_); - ASSERT_TRUE(s.ok()); - } - - std::string db_path_{"./test_db/flush_oldest_cf_test"}; - storage::StorageOptions options_; - storage::Storage db_; - uint32_t test_times_ = 100; - std::string key_ = "flush-oldest-cf-test"; - std::string key_prefix = "key_"; - std::string field_prefix_ = "field_"; - std::string value_prefix_ = "value_"; - rocksdb::WriteOptions write_options_; - rocksdb::ReadOptions read_options_; - LogQueue log_queue_; -}; - -TEST_F(FlushOldestCFTest, SimpleTest) { - const auto& rocksdb = db_.GetDBInstance(key_); - - auto add_kvs = [&](int start, int end) { - for (int i = start; i < end; i++) { - auto key = key_prefix + std::to_string(i); - auto v = value_prefix_ + std::to_string(i); - auto s = rocksdb->Set(key, v); - ASSERT_TRUE(s.ok()); - } - }; - - auto add_hash = [&](int start, int end) { - for (int i = start; i < end; i++) { - auto key = key_prefix + std::to_string(i); - auto v = value_prefix_ + std::to_string(i); - auto f = field_prefix_ + std::to_string(i); - int32_t res{}; - auto s = rocksdb->HSet(key, v, f, &res); - ASSERT_TRUE(s.ok()); - ASSERT_EQ(res, 1); - } - }; - - auto flush_cf = [&](size_t cf) { - auto s = rocksdb->GetDB()->Flush(rocksdb::FlushOptions(), rocksdb->GetColumnFamilyHandles()[cf]); - ASSERT_TRUE(s.ok()); - }; - - { - // type kv kv - // entry [1:1] -> ... [10:10] - // - // cf flushed_log_index flushed_sequence_number applied_log_index applied_sequence_number - // 0 0 0 10 10 - // other 0 0 0 0 - // - // last_flush_index log_index sequencenumber - // 0 0 - add_kvs(0, 10); - auto& last_flush_index = rocksdb->GetLogIndexOfColumnFamilies().GetLastFlushIndex(); - ASSERT_EQ(last_flush_index.log_index.load(), 0); - ASSERT_EQ(last_flush_index.seqno.load(), 0); - - auto& cf_0_status = rocksdb->GetLogIndexOfColumnFamilies().GetCFStatus(storage::kStringsCF); - ASSERT_EQ(cf_0_status.flushed_index.log_index, 0); - ASSERT_EQ(cf_0_status.flushed_index.seqno, 0); - ASSERT_EQ(cf_0_status.applied_index.log_index, 10); - ASSERT_EQ(cf_0_status.applied_index.seqno, 10); - - auto [smallest_applied_log_index_cf, smallest_applied_log_index, smallest_flushed_log_index_cf, - smallest_flushed_log_index, smallest_flushed_seqno] = - rocksdb->GetLogIndexOfColumnFamilies().GetSmallestLogIndex(-1); - - ASSERT_EQ(smallest_flushed_log_index, 0); - ASSERT_EQ(smallest_flushed_seqno, 0); - ASSERT_EQ(smallest_applied_log_index, 10); - auto size = rocksdb->GetCollector().GetSize(); - ASSERT_EQ(size, 10); - } - - { - // type kv kv hash hash hash - // entry [1:1] -> ... [10:10] -> [11:11] -> [12:13] -> ... -> [30:49] - // - // cf flushed_log_index flushed_sequence_number applied_log_index applied_sequence_number - // 0 0 0 10 10 - // 1 0 0 30 49 - // 2 0 0 30 50 - // other 0 0 0 0 - // - // last_flush_index log_index sequencenumber - // 0 0 - add_hash(10, 30); - auto& last_flush_index = rocksdb->GetLogIndexOfColumnFamilies().GetLastFlushIndex(); - ASSERT_EQ(last_flush_index.log_index.load(), 0); - ASSERT_EQ(last_flush_index.seqno.load(), 0); - - auto& cf_1_status = rocksdb->GetLogIndexOfColumnFamilies().GetCFStatus(storage::kHashesMetaCF); - ASSERT_EQ(cf_1_status.flushed_index.log_index, 0); - ASSERT_EQ(cf_1_status.flushed_index.seqno, 0); - ASSERT_EQ(cf_1_status.applied_index.log_index, 30); - ASSERT_EQ(cf_1_status.applied_index.seqno, 49); - - auto& cf_2_status = rocksdb->GetLogIndexOfColumnFamilies().GetCFStatus(storage::kHashesDataCF); - ASSERT_EQ(cf_2_status.flushed_index.log_index, 0); - ASSERT_EQ(cf_2_status.flushed_index.seqno, 0); - ASSERT_EQ(cf_2_status.applied_index.log_index, 30); - ASSERT_EQ(cf_2_status.applied_index.seqno, 50); - - auto [smallest_applied_log_index_cf, smallest_applied_log_index, smallest_flushed_log_index_cf, - smallest_flushed_log_index, smallest_flushed_seqno] = - rocksdb->GetLogIndexOfColumnFamilies().GetSmallestLogIndex(-1); - - ASSERT_EQ(smallest_flushed_log_index, 0); - ASSERT_EQ(smallest_flushed_seqno, 0); - ASSERT_EQ(smallest_applied_log_index, 10); - - auto size = rocksdb->GetCollector().GetSize(); - ASSERT_EQ(size, 30); - - auto is_pending_flush = rocksdb->GetCollector().IsFlushPending(); - ASSERT_TRUE(is_pending_flush); - } - - { - // type kv kv hash hash hash - // entry [1:1] -> ... [10:10] -> [11:11] -> [12:13] -> ... -> [30:49] - auto cur_par = rocksdb->GetCollector().GetList().begin(); - auto logindex = 1; - auto seq = 1; - for (int i = 1; i <= 10; i++) { - ASSERT_EQ(cur_par->GetAppliedLogIndex(), logindex); - ASSERT_EQ(cur_par->GetSequenceNumber(), seq); - cur_par = std::next(cur_par); - logindex++; - seq++; - } - - for (int i = 11; i <= 30; i++) { - ASSERT_EQ(cur_par->GetAppliedLogIndex(), logindex); - ASSERT_EQ(cur_par->GetSequenceNumber(), seq); - seq += 2; - logindex++; - cur_par = std::next(cur_par); - } - } - - { - // type kv kv hash hash hash - // entry [1:1] -> ... [10:10] -> [11:11] -> [12:13] -> ... -> [30:49] - // - // cf flushed_log_index flushed_sequence_number applied_log_index applied_sequence_number - // 0 0 0 10 10 - // 1 0 0 30 49 - // 2 0 0 30 50 - // other 0 0 0 0 - // - // last_flush_index log_index sequencenumber - // 0 0 - - auto gap = rocksdb->GetLogIndexOfColumnFamilies().GetPendingFlushGap(); - ASSERT_EQ(gap, 30); - flush_cf(1); - sleep(5); // sleep flush complete. - // 1) 根据 cf 1 的 latest SequenceNumber = 49 查到对应的 log index 为 30. 设置 cf 1 的 flushed_log_index 和 - // flushed_sequence_number 为 30 49. - // - // type kv kv hash hash hash - // entry [1:1] -> ... [10:10] -> [11:11] -> [12:13] -> ... -> [30:49] - // - // cf flushed_log_index flushed_sequence_number applied_log_index applied_sequence_number - // 0 0 0 10 10 - // 1 30 49 30 49 - // 2 0 0 30 50 - // other 0 0 0 0 - // - // last_flush_index log_index sequencenumber - // 0 0 - - // 2) 查找到此时的 smallest_applied_log_index_cf = 0 smallest_applied_log_index = 10 - // smallest_flushed_log_index_cf = 0 - // smallest_flushed_log_index = 0 smallest_flushed_seqno = 0 - // 根据 smallest_applied_log_index = 10 在队列长度 >= 2 的前提下, 持续删除 log_index < 10 的条目. - // - // type kv hash hash hash - // entry [10:10] -> [11:11] -> [12:13] -> ... -> [30:49] - // - // cf flushed_log_index flushed_sequence_number applied_log_index applied_sequence_number - // 0 0 0 10 10 - // 1 30 49 30 49 - // 2 0 0 30 50 - // other 0 0 0 0 - // - // last_flush_index log_index sequencenumber - // 0 0 - - // 3) 根据 smallest_flushed_log_index_cf = 0 smallest_flushed_log_index = 0 smallest_flushed_seqno = 0 - // 设置 last_flush_index 为 0, 0 - // - // type kv hash hash hash - // entry [10:10] -> [11:11] -> [12:13] -> ... -> [30:49] - // - // cf flushed_log_index flushed_sequence_number applied_log_index applied_sequence_number - // 0 0 0 10 10 - // 1 30 49 30 49 - // 2 0 0 30 50 - // other 0 0 0 0 - // - // last_flush_index log_index sequencenumber - // 0 0 - - // 4) 检测到队列中 logindex 的最大差值超过阈值, 触发 smallest_flushed_log_index_cf flush . 该 case 中对应 cf 为 0. - // 根据 cf 0 的 latest SequenceNumber = 10 查到对应的 log index 为 10. 设置 cf 0 的 flushed_log_index 和 - // flushed_sequence_number 为 10 10. - // - // type kv hash hash hash - // entry [10:10] -> [11:11] -> [12:13] -> ... -> [30:49] - // - // cf flushed_log_index flushed_sequence_number applied_log_index applied_sequence_number - // 0 10 10 10 10 - // 1 30 49 30 49 - // 2 0 0 30 50 - // other 0 0 0 0 - // - // last_flush_index log_index sequencenumber - // 0 0 - - // 5) 查找到此时的 smallest_applied_log_index_cf = 0 smallest_applied_log_index = 10 - // smallest_flushed_log_index_cf = 2 smallest_flushed_log_index = 0 smallest_flushed_seqno = 0 - // 根据 smallest_applied_log_index = 10 在队列长度 >= 2 的前提下, 删除 log_index < 10 的条目, 不变. - // - // type kv hash hash hash - // entry [10:10] -> [11:11] -> [12:13] -> ... -> [30:49] - // - // cf flushed_log_index flushed_sequence_number applied_log_index applied_sequence_number - // 0 10 10 10 10 - // 1 30 49 30 49 - // 2 0 0 30 50 - // other 0 0 0 0 - // - // last_flush_index log_index sequencenumber - // 0 0 - - // 6) 检测到队列中 logindex 的最大差值超过阈值, 触发 smallest_flushed_log_index_cf flush . 该 case 中对应 cf 为 2. - // 根据 cf 2 的 latest SequenceNumber = 50 查到对应的 log index 为 30. 设置 cf 2 的 flushed_log_index 和 - // flushed_sequence_number 为 30 50. - // - // type kv hash hash hash - // entry [10:10] -> [11:11] -> [12:13] -> ... -> [30:49] - // - // cf flushed_log_index flushed_sequence_number applied_log_index applied_sequence_number - // 0 10 10 10 10 - // 1 30 49 30 49 - // 2 30 50 30 50 - // other 0 0 0 0 - // - // last_flush_index log_index sequencenumber - // 0 0 - - // 7) 查找到此时的 smallest_applied_log_index_cf = 2 smallest_applied_log_index = 30 - // smallest_flushed_log_index_cf = 2 smallest_flushed_log_index = 30 smallest_flushed_seqno = 50 - // 根据 smallest_applied_log_index = 30 在队列长度 >= 2 的前提下, 删除 log_index < 50 的条目. - // - // type hash - // entry [30:49] - // - // cf flushed_log_index flushed_sequence_number applied_log_index applied_sequence_number - // 0 10 10 10 10 - // 1 30 49 30 49 - // 2 30 50 30 50 - // other 0 0 0 0 - // - // last_flush_index log_index sequencenumber - // 0 0 - - // 8) 根据 smallest_flushed_log_index_cf = 2 smallest_flushed_log_index = 30 smallest_flushed_seqno = 50 - // 设置 last_flush_index 为 30, 50. - // - // type hash - // entry [30:49] - // - // cf flushed_log_index flushed_sequence_number applied_log_index applied_sequence_number - // 0 10 10 10 10 - // 1 30 49 30 49 - // 2 30 50 30 50 - // other 0 0 0 0 - // - // last_flush_index log_index sequencenumber - // 30 50 - - // 9) 当设置 last_flush_index 为 30, 50 时, 会同时拉高没有数据的 cf 的 flushed_index, 该 case 为 cf 0, cf 1, - // 将 cf 0 的 flushed_index 从 10 10 提高为 30 50. - // 将 cf 1 的 flushed index 从 30 49 提升到 30 50. - // 其他没有写入的 cf flushed index 从 0 0 提升到 30 50. - // - // type hash - // entry [30:49] - // - // cf flushed_log_index flushed_sequence_number applied_log_index applied_sequence_number - // 0 30 50 10 10 - // 1 30 50 30 49 - // 2 30 50 30 50 - // other 30 50 0 0 - // - // last_flush_index log_index sequencenumber - // 30 50 - - // 9) 检测到队列长度未超过阈值, 结束 flush. - auto after_flush_size = rocksdb->GetCollector().GetSize(); - ASSERT_EQ(after_flush_size, 1); - - auto& cf_0_status = rocksdb->GetLogIndexOfColumnFamilies().GetCFStatus(storage::kStringsCF); - ASSERT_EQ(cf_0_status.flushed_index.log_index, 30); - ASSERT_EQ(cf_0_status.flushed_index.seqno, 50); - ASSERT_EQ(cf_0_status.applied_index.log_index, 10); - ASSERT_EQ(cf_0_status.applied_index.seqno, 10); - - auto& cf_1_status = rocksdb->GetLogIndexOfColumnFamilies().GetCFStatus(storage::kHashesMetaCF); - ASSERT_EQ(cf_1_status.flushed_index.log_index, 30); - ASSERT_EQ(cf_1_status.flushed_index.seqno, 50); - ASSERT_EQ(cf_1_status.applied_index.log_index, 30); - ASSERT_EQ(cf_1_status.applied_index.seqno, 49); - - auto& cf_2_status = rocksdb->GetLogIndexOfColumnFamilies().GetCFStatus(storage::kHashesDataCF); - ASSERT_EQ(cf_2_status.flushed_index.log_index, 30); - ASSERT_EQ(cf_2_status.flushed_index.seqno, 50); - ASSERT_EQ(cf_2_status.applied_index.log_index, 30); - ASSERT_EQ(cf_2_status.applied_index.seqno, 50); - - auto& last_flush_index = rocksdb->GetLogIndexOfColumnFamilies().GetLastFlushIndex(); - ASSERT_EQ(last_flush_index.log_index.load(), 30); - ASSERT_EQ(last_flush_index.seqno.load(), 50); - - auto& cf_3_status = rocksdb->GetLogIndexOfColumnFamilies().GetCFStatus(storage::kSetsMetaCF); - ASSERT_EQ(cf_3_status.flushed_index.log_index, 30); - ASSERT_EQ(cf_3_status.flushed_index.seqno, 50); - ASSERT_EQ(cf_3_status.applied_index.log_index, 0); - ASSERT_EQ(cf_3_status.applied_index.seqno, 0); - } - - { - add_kvs(30, 35); - // type hash -> kv -> ... -> kv - // entry [30:49] [31:51] [35:55] - // - // cf flushed_log_index flushed_sequence_number applied_log_index applied_sequence_number - // 0 30 50 35 55 - // 1 30 50 30 49 - // 2 30 50 30 50 - // other 30 50 0 0 - // - // last_flush_index log_index sequencenumber - // 30 50 - auto& last_flush_index = rocksdb->GetLogIndexOfColumnFamilies().GetLastFlushIndex(); - ASSERT_EQ(last_flush_index.log_index.load(), 30); - ASSERT_EQ(last_flush_index.seqno.load(), 50); - - auto& cf_0_status = rocksdb->GetLogIndexOfColumnFamilies().GetCFStatus(storage::kStringsCF); - ASSERT_EQ(cf_0_status.flushed_index.log_index, 30); - ASSERT_EQ(cf_0_status.flushed_index.seqno, 50); - ASSERT_EQ(cf_0_status.applied_index.log_index, 35); - ASSERT_EQ(cf_0_status.applied_index.seqno, 55); - - auto& cf_1_status = rocksdb->GetLogIndexOfColumnFamilies().GetCFStatus(storage::kHashesMetaCF); - ASSERT_EQ(cf_1_status.flushed_index.log_index, 30); - ASSERT_EQ(cf_1_status.flushed_index.seqno, 50); - ASSERT_EQ(cf_1_status.applied_index.log_index, 30); - ASSERT_EQ(cf_1_status.applied_index.seqno, 49); - - auto& cf_2_status = rocksdb->GetLogIndexOfColumnFamilies().GetCFStatus(storage::kHashesDataCF); - ASSERT_EQ(cf_2_status.flushed_index.log_index, 30); - ASSERT_EQ(cf_2_status.flushed_index.seqno, 50); - ASSERT_EQ(cf_2_status.applied_index.log_index, 30); - ASSERT_EQ(cf_2_status.applied_index.seqno, 50); - - auto& cf_3_status = rocksdb->GetLogIndexOfColumnFamilies().GetCFStatus(storage::kSetsMetaCF); - ASSERT_EQ(cf_3_status.flushed_index.log_index, 30); - ASSERT_EQ(cf_3_status.flushed_index.seqno, 50); - ASSERT_EQ(cf_3_status.applied_index.log_index, 0); - ASSERT_EQ(cf_3_status.applied_index.seqno, 0); - - auto [smallest_applied_log_index_cf, smallest_applied_log_index, smallest_flushed_log_index_cf, - smallest_flushed_log_index, smallest_flushed_seqno] = - rocksdb->GetLogIndexOfColumnFamilies().GetSmallestLogIndex(-1); - - // 除了 cf 0 之外, 其余的 cf 都没有未持久化数据, 所以不在我们统计范围之内. - ASSERT_EQ(smallest_applied_log_index_cf, 0); - ASSERT_EQ(smallest_applied_log_index, 35); - - ASSERT_EQ(smallest_flushed_log_index_cf, 0); - ASSERT_EQ(smallest_flushed_log_index, 30); - ASSERT_EQ(smallest_flushed_seqno, 50); - - auto size = rocksdb->GetCollector().GetSize(); - ASSERT_EQ(size, 6); - - auto is_pending_flush = rocksdb->GetCollector().IsFlushPending(); - ASSERT_TRUE(!is_pending_flush); - } -}; diff --git a/src/storage/tests/log_index_test.cc b/src/storage/tests/log_index_test.cc deleted file mode 100644 index 54e656979..000000000 --- a/src/storage/tests/log_index_test.cc +++ /dev/null @@ -1,274 +0,0 @@ -/* - * Copyright (c) 2024-present, Qihoo, Inc. All rights reserved. - * This source code is licensed under the BSD-style license found in the - * LICENSE file in the root directory of this source tree. An additional grant - * of patent rights can be found in the PATENTS file in the same directory. - */ - -#include -#include -#include -#include -#include -#include - -#include "fmt/core.h" -#include "gtest/gtest.h" -#include "rocksdb/db.h" -#include "rocksdb/listener.h" -#include "rocksdb/metadata.h" -#include "rocksdb/options.h" - -#include "pstd/log.h" -#include "pstd/thread_pool.h" -#include "src/log_index.h" -#include "src/redis.h" -#include "storage/storage.h" -#include "storage/util.h" - -using namespace storage; // NOLINT - -class LogIniter { - public: - LogIniter() { - logger::Init("./log_index_test.log"); - spdlog::set_level(spdlog::level::info); - } -}; -static LogIniter initer; - -TEST(TablePropertyTest, SimpleTest) { - constexpr const char* kDbPath = "./log_index_test_db"; - rocksdb::Options options; - options.create_if_missing = true; - LogIndexAndSequenceCollector collector; - options.table_properties_collector_factories.push_back( - std::make_shared(collector)); - rocksdb::DB* db{nullptr}; - auto s = rocksdb::DB::Open(options, kDbPath, &db); - EXPECT_TRUE(s.ok()); - - std::string key = "table-property-test"; - s = db->Put(rocksdb::WriteOptions(), key, key); - EXPECT_TRUE(s.ok()); - std::string res; - s = db->Get(rocksdb::ReadOptions(), key, &res); - EXPECT_TRUE(s.ok()); - EXPECT_EQ(key, res); - collector.Update(233333, db->GetLatestSequenceNumber()); - db->Flush(rocksdb::FlushOptions()); - - rocksdb::TablePropertiesCollection properties; - s = db->GetPropertiesOfAllTables(&properties); - EXPECT_TRUE(s.ok()); - EXPECT_TRUE(properties.size() == 1); - for (auto& [name, prop] : properties) { - const auto& collector = prop->user_collected_properties; - auto it = collector.find(static_cast(LogIndexTablePropertiesCollector::kPropertyName)); - EXPECT_NE(it, collector.cend()); - EXPECT_EQ(it->second, "233333/" + std::to_string(db->GetLatestSequenceNumber())); - } - - db->Close(); - DeleteFiles(kDbPath); -} - -class LogQueue : public pstd::noncopyable { - public: - using WriteCallback = std::function; - - explicit LogQueue(WriteCallback&& cb) : write_cb_(std::move(cb)) { consumer_.SetMaxIdleThread(1); } - - void AppendLog(const pikiwidb::Binlog& log, std::promise&& promise) { - auto task = [&] { - auto idx = next_log_idx_.fetch_add(1); - auto s = write_cb_(log, idx); - promise.set_value(s); - }; - consumer_.ExecuteTask(std::move(task)); - } - - private: - WriteCallback write_cb_ = nullptr; - pstd::ThreadPool consumer_; - std::atomic next_log_idx_{1}; -}; - -class LogIndexTest : public ::testing::Test { - public: - LogIndexTest() - : log_queue_([this](const pikiwidb::Binlog& log, LogIndex log_idx) { return db_.OnBinlogWrite(log, log_idx); }) { - options_.options.create_if_missing = true; - options_.db_instance_num = 1; - options_.raft_timeout_s = 10000; - options_.append_log_function = [this](const pikiwidb::Binlog& log, std::promise&& promise) { - log_queue_.AppendLog(log, std::move(promise)); - }; - options_.do_snapshot_function = [](int64_t log_index, bool sync) {}; - } - ~LogIndexTest() override { DeleteFiles(db_path_.c_str()); } - - void SetUp() override { - if (access(db_path_.c_str(), F_OK) == 0) { - std::filesystem::remove_all(db_path_.c_str()); - } - mkdir(db_path_.c_str(), 0755); - auto s = db_.Open(options_, db_path_); - ASSERT_TRUE(s.ok()); - } - - std::string db_path_{"./test_db/log_index_test"}; - StorageOptions options_; - Storage db_; - uint32_t test_times_ = 100; - std::string key_ = "log-index-test"; - std::string field_prefix_ = "field"; - std::string value_prefix_ = "value"; - rocksdb::WriteOptions write_options_; - rocksdb::ReadOptions read_options_; - LogQueue log_queue_; - - auto CreateRandomKey(int i, size_t length) -> std::string { - auto res = CreateRandomFieldValue(i, length); - res.append(key_); - return res; - } - static auto CreateRandomFieldValue(int i, size_t length) -> std::string { - std::mt19937 gen(i); - std::string str(length, 0); - for (int i = 0; i < length; i++) { - str[i] = chars[gen() % (sizeof(chars) / sizeof(char))]; - } - return str; - } - constexpr static char chars[] = {'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j', 'k', 'l', 'm', 'n', 'o', 'p', - 'q', 'r', 's', 't', 'u', 'v', 'w', 'x', 'y', 'z', 'A', 'B', 'C', 'D', 'E', 'F', - 'G', 'H', 'I', 'J', 'K', 'L', 'M', 'N', 'O', 'P', 'Q', 'R', 'S', 'T', 'U', 'V', - 'W', 'X', 'Y', 'Z', '0', '1', '2', '3', '4', '5', '6', '7', '8', '9'}; -}; - -TEST_F(LogIndexTest, DoNothing) {} - -TEST_F(LogIndexTest, SimpleTest) { // NOLINT - auto& redis = db_.GetDBInstance(key_); - auto add_kvs = [&](int start, int end) { - for (int i = start; i < end; i++) { - auto key = CreateRandomKey(i, 256); - auto fv = CreateRandomFieldValue(i, 512); - int32_t res{}; - auto s = redis->HSet(key, fv, fv, &res); - ASSERT_TRUE(s.ok()); - ASSERT_EQ(1, res); - - std::string get_res; - s = redis->HGet(key, fv, &get_res); - ASSERT_TRUE(s.ok()); - ASSERT_EQ(fv, get_res); - } - }; - auto flushdb = [&]() { - auto s = redis->GetDB()->Flush(rocksdb::FlushOptions(), redis->GetColumnFamilyHandles()[kHashesMetaCF]); - ASSERT_TRUE(s.ok()); - s = redis->GetDB()->Flush(rocksdb::FlushOptions(), redis->GetColumnFamilyHandles()[kHashesDataCF]); - ASSERT_TRUE(s.ok()); - }; - - // one key test - { - add_kvs(0, 1); - flushdb(); - - rocksdb::TablePropertiesCollection properties; - auto s = redis->GetDB()->GetPropertiesOfAllTables(redis->GetColumnFamilyHandles()[kHashesMetaCF], &properties); - ASSERT_TRUE(s.ok()); - ASSERT_TRUE(properties.size() == 1); - auto res = LogIndexTablePropertiesCollector::GetLargestLogIndexFromTableCollection(properties); - EXPECT_TRUE(res.has_value()); - assert(res.has_value()); - EXPECT_EQ(res->GetAppliedLogIndex(), 1); - EXPECT_EQ(res->GetSequenceNumber(), 1); - - properties.clear(); - s = redis->GetDB()->GetPropertiesOfAllTables(redis->GetColumnFamilyHandles()[kHashesDataCF], &properties); - ASSERT_TRUE(s.ok()); - ASSERT_TRUE(properties.size() == 1); - res = LogIndexTablePropertiesCollector::GetLargestLogIndexFromTableCollection(properties); - EXPECT_TRUE(res.has_value()); - assert(res.has_value()); - EXPECT_EQ(res->GetAppliedLogIndex(), 1); - EXPECT_EQ(res->GetSequenceNumber(), 2); - } - - // more keys - { - add_kvs(1, 10000); - flushdb(); - - rocksdb::TablePropertiesCollection properties; - auto s = redis->GetDB()->GetPropertiesOfAllTables(redis->GetColumnFamilyHandles()[kHashesMetaCF], &properties); - ASSERT_TRUE(s.ok()); - auto res = LogIndexTablePropertiesCollector::GetLargestLogIndexFromTableCollection(properties); - EXPECT_TRUE(res.has_value()); - assert(res.has_value()); - EXPECT_EQ(res->GetAppliedLogIndex(), 10000); - EXPECT_EQ(res->GetSequenceNumber(), 19999); - - properties.clear(); - s = redis->GetDB()->GetPropertiesOfAllTables(redis->GetColumnFamilyHandles()[kHashesDataCF], &properties); - ASSERT_TRUE(s.ok()); - res = LogIndexTablePropertiesCollector::GetLargestLogIndexFromTableCollection(properties); - EXPECT_TRUE(res.has_value()); - assert(res.has_value()); - EXPECT_EQ(res->GetAppliedLogIndex(), 10000); - EXPECT_EQ(res->GetSequenceNumber(), 20000); - } - - // more flush - { - for (int i = 1; i < 20; i++) { - fmt::println("==================i={} start==========================", i); - auto start = i * 10000; - auto end = start + 10000; - - add_kvs(start, end); - flushdb(); - // sleep(1); - - { - rocksdb::TablePropertiesCollection properties; - auto s = redis->GetDB()->GetPropertiesOfAllTables(redis->GetColumnFamilyHandles()[kHashesMetaCF], &properties); - s = redis->GetDB()->GetPropertiesOfAllTables(redis->GetColumnFamilyHandles()[kHashesDataCF], &properties); - std::vector metas; - redis->GetDB()->GetLiveFilesMetaData(&metas); - for (const auto& meta : metas) { - auto file = meta.directory + meta.name; - if (!properties.contains(file)) { - fmt::println("{}: L{}, {}, not contains", file, meta.level, meta.column_family_name); - continue; - } - auto res = LogIndexTablePropertiesCollector::ReadStatsFromTableProps(properties.at(file)); - assert(res.has_value()); - fmt::println("{}: L{}, {}, logidx={}", file, meta.level, meta.column_family_name, res->GetAppliedLogIndex()); - } - } - - rocksdb::TablePropertiesCollection properties; - auto s = redis->GetDB()->GetPropertiesOfAllTables(redis->GetColumnFamilyHandles()[kHashesMetaCF], &properties); - ASSERT_TRUE(s.ok()); - auto res = LogIndexTablePropertiesCollector::GetLargestLogIndexFromTableCollection(properties); - EXPECT_TRUE(res.has_value()); - assert(res.has_value()); - EXPECT_EQ(res->GetAppliedLogIndex(), end); - EXPECT_EQ(res->GetSequenceNumber(), end * 2 - 1); - - properties.clear(); - s = redis->GetDB()->GetPropertiesOfAllTables(redis->GetColumnFamilyHandles()[kHashesDataCF], &properties); - ASSERT_TRUE(s.ok()); - res = LogIndexTablePropertiesCollector::GetLargestLogIndexFromTableCollection(properties); - EXPECT_TRUE(res.has_value()); - assert(res.has_value()); - EXPECT_EQ(res->GetAppliedLogIndex(), end); - EXPECT_EQ(res->GetSequenceNumber(), end * 2); - } - } -} diff --git a/tests/key_test.go b/tests/key_test.go index ff457ffc0..7be66a170 100644 --- a/tests/key_test.go +++ b/tests/key_test.go @@ -266,21 +266,13 @@ var _ = Describe("Keyspace", Ordered, func() { time.Sleep(5 * time.Second) Expect(client.Exists(ctx, DefaultKey).Val()).To(Equal(int64(1))) - // multi data type - Expect(client.LPush(ctx, DefaultKey, "l").Err()).NotTo(HaveOccurred()) - Expect(client.HSet(ctx, DefaultKey, "h", "h").Err()).NotTo(HaveOccurred()) - Expect(client.SAdd(ctx, DefaultKey, "s").Err()).NotTo(HaveOccurred()) - Expect(client.ZAdd(ctx, DefaultKey, redis.Z{Score: 1, Member: "z"}).Err()).NotTo(HaveOccurred()) - Expect(client.Set(ctx, DefaultKey, DefaultValue, 0).Val()).To(Equal(OK)) - Expect(client.PExpireAt(ctx, DefaultKey, time.Now().Add(time.Second*1000)).Err()).NotTo(HaveOccurred()) - Expect(client.Persist(ctx, DefaultKey).Err()).NotTo(HaveOccurred()) - // del keys Expect(client.PExpireAt(ctx, DefaultKey, time.Now().Add(time.Second*1)).Err()).NotTo(HaveOccurred()) time.Sleep(2 * time.Second) }) It("keys", func() { + Expect(client.Del(ctx, "a1", "k1", "k2", "k3", "k4", "k5", "key").Err()).NotTo(HaveOccurred()) // empty Expect(client.Keys(ctx, "*").Val()).To(Equal([]string{})) Expect(client.Keys(ctx, "dummy").Val()).To(Equal([]string{})) @@ -294,10 +286,10 @@ var _ = Describe("Keyspace", Ordered, func() { Expect(client.ZAdd(ctx, "k5", redis.Z{Score: 1, Member: "v5"}).Val()).To(Equal(int64(1))) // all - Expect(client.Keys(ctx, "*").Val()).To(Equal([]string{"a1", "k1", "k3", "k4", "k5", "k2"})) + Expect(client.Keys(ctx, "*").Val()).To(Equal([]string{"a1", "k1", "k2", "k3", "k4", "k5"})) // pattern - Expect(client.Keys(ctx, "k*").Val()).To(Equal([]string{"k1", "k3", "k4", "k5", "k2"})) + Expect(client.Keys(ctx, "k*").Val()).To(Equal([]string{"k1", "k2", "k3", "k4", "k5"})) Expect(client.Keys(ctx, "k1").Val()).To(Equal([]string{"k1"})) // del keys diff --git a/tests/list_test.go b/tests/list_test.go index 2a794f404..d4101ef7e 100644 --- a/tests/list_test.go +++ b/tests/list_test.go @@ -77,101 +77,101 @@ var _ = Describe("List", Ordered, func() { It("Cmd LPUSH", func() { log.Println("Cmd LPUSH Begin") - Expect(client.LPush(ctx, DefaultKey, s2s["key_2"]).Val()).To(Equal(int64(1))) - Expect(client.LPush(ctx, DefaultKey, s2s["key_1"]).Val()).To(Equal(int64(2))) + Expect(client.LPush(ctx, "list", s2s["key_2"]).Val()).To(Equal(int64(1))) + Expect(client.LPush(ctx, "list", s2s["key_1"]).Val()).To(Equal(int64(2))) - Expect(client.LRange(ctx, DefaultKey, 0, -1).Val()).To(Equal([]string{s2s["key_1"], s2s["key_2"]})) + Expect(client.LRange(ctx, "list", 0, -1).Val()).To(Equal([]string{s2s["key_1"], s2s["key_2"]})) //del - del := client.Del(ctx, DefaultKey) + del := client.Del(ctx, "list") Expect(del.Err()).NotTo(HaveOccurred()) }) It("Cmd LPUSHX", func() { - Expect(client.LPushX(ctx, DefaultKey, s2s["key_1"]).Val()).To(Equal(int64(0))) - Expect(client.LRange(ctx, DefaultKey, 0, -1).Val()).To(Equal([]string{})) + Expect(client.LPushX(ctx, "list", s2s["key_1"]).Val()).To(Equal(int64(0))) + Expect(client.LRange(ctx, "list", 0, -1).Val()).To(Equal([]string{})) - Expect(client.LPush(ctx, DefaultKey, s2s["key_2"]).Val()).To(Equal(int64(1))) - Expect(client.LPushX(ctx, DefaultKey, s2s["key_3"]).Val()).To(Equal(int64(2))) - Expect(client.LRange(ctx, DefaultKey, 0, -1).Val()).To(Equal([]string{s2s["key_3"], s2s["key_2"]})) + Expect(client.LPush(ctx, "list", s2s["key_2"]).Val()).To(Equal(int64(1))) + Expect(client.LPushX(ctx, "list", s2s["key_3"]).Val()).To(Equal(int64(2))) + Expect(client.LRange(ctx, "list", 0, -1).Val()).To(Equal([]string{s2s["key_3"], s2s["key_2"]})) - Expect(client.LPushX(ctx, DefaultKey, s2s["key_4"], s2s["key_5"]).Val()).To(Equal(int64(4))) - Expect(client.LRange(ctx, DefaultKey, 0, -1).Val()).To(Equal([]string{s2s["key_5"], s2s["key_4"], s2s["key_3"], s2s["key_2"]})) + Expect(client.LPushX(ctx, "list", s2s["key_4"], s2s["key_5"]).Val()).To(Equal(int64(4))) + Expect(client.LRange(ctx, "list", 0, -1).Val()).To(Equal([]string{s2s["key_5"], s2s["key_4"], s2s["key_3"], s2s["key_2"]})) - del := client.Del(ctx, DefaultKey) + del := client.Del(ctx, "list") Expect(del.Err()).NotTo(HaveOccurred()) }) It("Cmd RPUSH", func() { log.Println("Cmd RPUSH Begin") - Expect(client.RPush(ctx, DefaultKey, s2s["key_1"]).Val()).To(Equal(int64(1))) - Expect(client.RPush(ctx, DefaultKey, s2s["key_2"]).Val()).To(Equal(int64(2))) + Expect(client.RPush(ctx, "list", s2s["key_1"]).Val()).To(Equal(int64(1))) + Expect(client.RPush(ctx, "list", s2s["key_2"]).Val()).To(Equal(int64(2))) - Expect(client.LRange(ctx, DefaultKey, 0, -1).Val()).To(Equal([]string{s2s["key_1"], s2s["key_2"]})) + Expect(client.LRange(ctx, "list", 0, -1).Val()).To(Equal([]string{s2s["key_1"], s2s["key_2"]})) //del - del := client.Del(ctx, DefaultKey) + del := client.Del(ctx, "list") Expect(del.Err()).NotTo(HaveOccurred()) }) It("Cmd RPUSHX", func() { - Expect(client.RPushX(ctx, DefaultKey, s2s["key_1"]).Val()).To(Equal(int64(0))) - Expect(client.LRange(ctx, DefaultKey, 0, -1).Val()).To(Equal([]string{})) + Expect(client.RPushX(ctx, "list", s2s["key_1"]).Val()).To(Equal(int64(0))) + Expect(client.LRange(ctx, "list", 0, -1).Val()).To(Equal([]string{})) - Expect(client.RPush(ctx, DefaultKey, s2s["key_2"]).Val()).To(Equal(int64(1))) - Expect(client.RPushX(ctx, DefaultKey, s2s["key_3"]).Val()).To(Equal(int64(2))) - Expect(client.LRange(ctx, DefaultKey, 0, -1).Val()).To(Equal([]string{s2s["key_2"], s2s["key_3"]})) + Expect(client.RPush(ctx, "list", s2s["key_2"]).Val()).To(Equal(int64(1))) + Expect(client.RPushX(ctx, "list", s2s["key_3"]).Val()).To(Equal(int64(2))) + Expect(client.LRange(ctx, "list", 0, -1).Val()).To(Equal([]string{s2s["key_2"], s2s["key_3"]})) - Expect(client.RPushX(ctx, DefaultKey, s2s["key_4"], s2s["key_5"]).Val()).To(Equal(int64(4))) - Expect(client.LRange(ctx, DefaultKey, 0, -1).Val()).To(Equal([]string{s2s["key_2"], s2s["key_3"], s2s["key_4"], s2s["key_5"]})) + Expect(client.RPushX(ctx, "list", s2s["key_4"], s2s["key_5"]).Val()).To(Equal(int64(4))) + Expect(client.LRange(ctx, "list", 0, -1).Val()).To(Equal([]string{s2s["key_2"], s2s["key_3"], s2s["key_4"], s2s["key_5"]})) - del := client.Del(ctx, DefaultKey) + del := client.Del(ctx, "list") Expect(del.Err()).NotTo(HaveOccurred()) }) It("Cmd LPop", func() { - rPush := client.RPush(ctx, DefaultKey, s2s["key_1"]) + rPush := client.RPush(ctx, "list", s2s["key_1"]) Expect(rPush.Err()).NotTo(HaveOccurred()) - rPush = client.RPush(ctx, DefaultKey, s2s["key_2"]) + rPush = client.RPush(ctx, "list", s2s["key_2"]) Expect(rPush.Err()).NotTo(HaveOccurred()) - rPush = client.RPush(ctx, DefaultKey, s2s["key_3"]) + rPush = client.RPush(ctx, "list", s2s["key_3"]) Expect(rPush.Err()).NotTo(HaveOccurred()) - lPop := client.LPop(ctx, DefaultKey) + lPop := client.LPop(ctx, "list") Expect(lPop.Err()).NotTo(HaveOccurred()) Expect(lPop.Val()).To(Equal(s2s["key_1"])) - lRange := client.LRange(ctx, DefaultKey, 0, -1) + lRange := client.LRange(ctx, "list", 0, -1) Expect(lRange.Err()).NotTo(HaveOccurred()) Expect(lRange.Val()).To(Equal([]string{s2s["key_2"], s2s["key_3"]})) - err := client.Do(ctx, "LPOP", DefaultKey, 1, 2).Err() + err := client.Do(ctx, "LPOP", "list", 1, 2).Err() Expect(err).To(MatchError(ContainSubstring("ERR wrong number of arguments for 'lpop' command"))) - del := client.Del(ctx, DefaultKey) + del := client.Del(ctx, "list") Expect(del.Err()).NotTo(HaveOccurred()) }) It("should RPop", func() { - rPush := client.RPush(ctx, DefaultKey, s2s["key_1"]) + rPush := client.RPush(ctx, "list", s2s["key_1"]) Expect(rPush.Err()).NotTo(HaveOccurred()) - rPush = client.RPush(ctx, DefaultKey, s2s["key_2"]) + rPush = client.RPush(ctx, "list", s2s["key_2"]) Expect(rPush.Err()).NotTo(HaveOccurred()) - rPush = client.RPush(ctx, DefaultKey, s2s["key_3"]) + rPush = client.RPush(ctx, "list", s2s["key_3"]) Expect(rPush.Err()).NotTo(HaveOccurred()) - rPop := client.RPop(ctx, DefaultKey) + rPop := client.RPop(ctx, "list") Expect(rPop.Err()).NotTo(HaveOccurred()) Expect(rPop.Val()).To(Equal(s2s["key_3"])) - lRange := client.LRange(ctx, DefaultKey, 0, -1) + lRange := client.LRange(ctx, "list", 0, -1) Expect(lRange.Err()).NotTo(HaveOccurred()) Expect(lRange.Val()).To(Equal([]string{s2s["key_1"], s2s["key_2"]})) - err := client.Do(ctx, "RPOP", DefaultKey, 1, 2).Err() + err := client.Do(ctx, "RPOP", "list", 1, 2).Err() Expect(err).To(MatchError(ContainSubstring("ERR wrong number of arguments for 'rpop' command"))) //del - del := client.Del(ctx, DefaultKey) + del := client.Del(ctx, "list") Expect(del.Err()).NotTo(HaveOccurred()) }) @@ -201,141 +201,141 @@ var _ = Describe("List", Ordered, func() { }) It("Cmd LRem", func() { - rPush := client.RPush(ctx, DefaultKey, s2s["key_1"]) + rPush := client.RPush(ctx, "list", s2s["key_1"]) Expect(rPush.Err()).NotTo(HaveOccurred()) - rPush = client.RPush(ctx, DefaultKey, s2s["key_1"]) + rPush = client.RPush(ctx, "list", s2s["key_1"]) Expect(rPush.Err()).NotTo(HaveOccurred()) - rPush = client.RPush(ctx, DefaultKey, s2s["key_2"]) + rPush = client.RPush(ctx, "list", s2s["key_2"]) Expect(rPush.Err()).NotTo(HaveOccurred()) - rPush = client.RPush(ctx, DefaultKey, s2s["key_1"]) + rPush = client.RPush(ctx, "list", s2s["key_1"]) Expect(rPush.Err()).NotTo(HaveOccurred()) - lRem := client.LRem(ctx, DefaultKey, -2, s2s["key_1"]) + lRem := client.LRem(ctx, "list", -2, s2s["key_1"]) Expect(lRem.Err()).NotTo(HaveOccurred()) Expect(lRem.Val()).To(Equal(int64(2))) - lRange := client.LRange(ctx, DefaultKey, 0, -1) + lRange := client.LRange(ctx, "list", 0, -1) Expect(lRange.Err()).NotTo(HaveOccurred()) Expect(lRange.Val()).To(Equal([]string{s2s["key_1"], s2s["key_2"]})) //del - del := client.Del(ctx, DefaultKey) + del := client.Del(ctx, "list") Expect(del.Err()).NotTo(HaveOccurred()) }) It("should LTrim", func() { - rPush := client.RPush(ctx, DefaultKey, s2s["key_1"]) + rPush := client.RPush(ctx, "list", s2s["key_1"]) Expect(rPush.Err()).NotTo(HaveOccurred()) - rPush = client.RPush(ctx, DefaultKey, s2s["key_2"]) + rPush = client.RPush(ctx, "list", s2s["key_2"]) Expect(rPush.Err()).NotTo(HaveOccurred()) - rPush = client.RPush(ctx, DefaultKey, s2s["key_3"]) + rPush = client.RPush(ctx, "list", s2s["key_3"]) Expect(rPush.Err()).NotTo(HaveOccurred()) - lTrim := client.LTrim(ctx, DefaultKey, 1, -1) + lTrim := client.LTrim(ctx, "list", 1, -1) Expect(lTrim.Err()).NotTo(HaveOccurred()) Expect(lTrim.Val()).To(Equal(OK)) - lRange := client.LRange(ctx, DefaultKey, 0, -1) + lRange := client.LRange(ctx, "list", 0, -1) Expect(lRange.Err()).NotTo(HaveOccurred()) Expect(lRange.Val()).To(Equal([]string{s2s["key_2"], s2s["key_3"]})) // del - del := client.Del(ctx, DefaultKey) + del := client.Del(ctx, "list") Expect(del.Err()).NotTo(HaveOccurred()) }) It("should LSet", func() { - rPush := client.RPush(ctx, DefaultKey, s2s["key_1"]) + rPush := client.RPush(ctx, "list", s2s["key_1"]) Expect(rPush.Err()).NotTo(HaveOccurred()) - rPush = client.RPush(ctx, DefaultKey, s2s["key_2"]) + rPush = client.RPush(ctx, "list", s2s["key_2"]) Expect(rPush.Err()).NotTo(HaveOccurred()) - rPush = client.RPush(ctx, DefaultKey, s2s["key_3"]) + rPush = client.RPush(ctx, "list", s2s["key_3"]) Expect(rPush.Err()).NotTo(HaveOccurred()) - lSet := client.LSet(ctx, DefaultKey, 0, s2s["key_4"]) + lSet := client.LSet(ctx, "list", 0, s2s["key_4"]) Expect(lSet.Err()).NotTo(HaveOccurred()) Expect(lSet.Val()).To(Equal(OK)) - lSet = client.LSet(ctx, DefaultKey, -2, s2s["key_5"]) + lSet = client.LSet(ctx, "list", -2, s2s["key_5"]) Expect(lSet.Err()).NotTo(HaveOccurred()) Expect(lSet.Val()).To(Equal(OK)) - lRange := client.LRange(ctx, DefaultKey, 0, -1) + lRange := client.LRange(ctx, "list", 0, -1) Expect(lRange.Err()).NotTo(HaveOccurred()) Expect(lRange.Val()).To(Equal([]string{s2s["key_4"], s2s["key_5"], s2s["key_3"]})) // del - del := client.Del(ctx, DefaultKey) + del := client.Del(ctx, "list") Expect(del.Err()).NotTo(HaveOccurred()) }) It("should LInsert", func() { - rPush := client.RPush(ctx, DefaultKey, s2s["key_1"]) + rPush := client.RPush(ctx, "list", s2s["key_1"]) Expect(rPush.Err()).NotTo(HaveOccurred()) - rPush = client.RPush(ctx, DefaultKey, s2s["key_2"]) + rPush = client.RPush(ctx, "list", s2s["key_2"]) Expect(rPush.Err()).NotTo(HaveOccurred()) - lInsert := client.LInsert(ctx, DefaultKey, "BEFORE", s2s["key_2"], s2s["key_3"]) + lInsert := client.LInsert(ctx, "list", "BEFORE", s2s["key_2"], s2s["key_3"]) Expect(lInsert.Err()).NotTo(HaveOccurred()) Expect(lInsert.Val()).To(Equal(int64(3))) - lRange := client.LRange(ctx, DefaultKey, 0, -1) + lRange := client.LRange(ctx, "list", 0, -1) Expect(lRange.Err()).NotTo(HaveOccurred()) Expect(lRange.Val()).To(Equal([]string{s2s["key_1"], s2s["key_3"], s2s["key_2"]})) // del - del := client.Del(ctx, DefaultKey) + del := client.Del(ctx, "list") Expect(del.Err()).NotTo(HaveOccurred()) }) It("SHOULD LIndex", func() { - rPush := client.RPush(ctx, DefaultKey, s2s["key_1"]) + rPush := client.RPush(ctx, "list", s2s["key_1"]) Expect(rPush.Err()).NotTo(HaveOccurred()) - rPush = client.RPush(ctx, DefaultKey, s2s["key_2"]) + rPush = client.RPush(ctx, "list", s2s["key_2"]) Expect(rPush.Err()).NotTo(HaveOccurred()) - rPush = client.RPush(ctx, DefaultKey, s2s["key_3"]) + rPush = client.RPush(ctx, "list", s2s["key_3"]) Expect(rPush.Err()).NotTo(HaveOccurred()) - lIndex := client.LIndex(ctx, DefaultKey, 0) + lIndex := client.LIndex(ctx, "list", 0) Expect(lIndex.Err()).NotTo(HaveOccurred()) Expect(lIndex.Val()).To(Equal(s2s["key_1"])) - lIndex = client.LIndex(ctx, DefaultKey, 1) + lIndex = client.LIndex(ctx, "list", 1) Expect(lIndex.Err()).NotTo(HaveOccurred()) Expect(lIndex.Val()).To(Equal(s2s["key_2"])) - lIndex = client.LIndex(ctx, DefaultKey, -1) + lIndex = client.LIndex(ctx, "list", -1) Expect(lIndex.Err()).NotTo(HaveOccurred()) Expect(lIndex.Val()).To(Equal(s2s["key_3"])) - lIndex = client.LIndex(ctx, DefaultKey, 4) + lIndex = client.LIndex(ctx, "list", 4) Expect(lIndex.Err()).To(Equal(redis.Nil)) Expect(lIndex.Val()).To(Equal("")) - err := client.Do(ctx, "lindex", DefaultKey, 1, 2).Err() + err := client.Do(ctx, "lindex", "list", 1, 2).Err() Expect(err).To(MatchError(ContainSubstring("ERR wrong number of arguments for 'lindex' command"))) - del := client.Del(ctx, DefaultKey) + del := client.Del(ctx, "list") Expect(del.Err()).NotTo(HaveOccurred()) }) It("SHOULD LLen", func() { - lLen := client.LLen(ctx, DefaultKey) + lLen := client.LLen(ctx, "list") Expect(lLen.Err()).NotTo(HaveOccurred()) Expect(lLen.Val()).To(Equal(int64(0))) - rPush := client.RPush(ctx, DefaultKey, s2s["key_1"]) + rPush := client.RPush(ctx, "list", s2s["key_1"]) Expect(rPush.Err()).NotTo(HaveOccurred()) - rPush = client.RPush(ctx, DefaultKey, s2s["key_2"]) + rPush = client.RPush(ctx, "list", s2s["key_2"]) Expect(rPush.Err()).NotTo(HaveOccurred()) - lLen = client.LLen(ctx, DefaultKey) + lLen = client.LLen(ctx, "list") Expect(lLen.Err()).NotTo(HaveOccurred()) Expect(lLen.Val()).To(Equal(int64(2))) - err := client.Do(ctx, "llen", DefaultKey, 1).Err() + err := client.Do(ctx, "llen", "list", 1).Err() Expect(err).To(MatchError(ContainSubstring("ERR wrong number of arguments for 'llen' command"))) - del := client.Del(ctx, DefaultKey) + del := client.Del(ctx, "list") Expect(del.Err()).NotTo(HaveOccurred()) }) })