diff --git a/.gitignore b/.gitignore index 1381333c3..c7b52255d 100644 --- a/.gitignore +++ b/.gitignore @@ -8,6 +8,7 @@ clang-* third_party genfiles/* *.sublime-* +*.orig .tags !third_party/include/* *.pyc diff --git a/src/core/json/jsonpath_test.cc b/src/core/json/jsonpath_test.cc index 28e23d843..8adfccfaa 100644 --- a/src/core/json/jsonpath_test.cc +++ b/src/core/json/jsonpath_test.cc @@ -249,4 +249,19 @@ TEST_F(JsonPathTest, EvalDescent) { ASSERT_THAT(arr, ElementsAre(json_type::array_value, json_type::object_value)); } +TEST_F(JsonPathTest, Wildcard) { + ASSERT_EQ(0, Parse("$[*]")); + Path path = driver_.TakePath(); + ASSERT_EQ(1, path.size()); + EXPECT_THAT(path[0], SegType(SegmentType::WILDCARD)); + + JsonType json = JsonFromString(R"([1, 2, 3])").value(); + vector arr; + EvaluatePath(path, json, [&](optional key, const JsonType& val) { + ASSERT_FALSE(key); + arr.push_back(val.as()); + }); + ASSERT_THAT(arr, ElementsAre(1, 2, 3)); +} + } // namespace dfly::json diff --git a/src/core/json/path.cc b/src/core/json/path.cc index c3f2e78a0..eccfe1ead 100644 --- a/src/core/json/path.cc +++ b/src/core/json/path.cc @@ -152,7 +152,7 @@ auto Dfs::Item::Init(const PathSegment& segment) -> AdvanceResult { if (segment_step_ == 1) { // first time, branching to return the same object but with the next segment, // exploring the path of ignoring the DESCENT operator. - // Alsom, shift the state (segment_step) to bypass this branch next time. + // Also, shift the state (segment_step) to bypass this branch next time. segment_step_ = 0; return DepthState{depth_state_.first, depth_state_.second + 1}; } diff --git a/src/facade/facade_test.h b/src/facade/facade_test.h index 922c7da27..4dfdeb795 100644 --- a/src/facade/facade_test.h +++ b/src/facade/facade_test.h @@ -78,9 +78,9 @@ inline ::testing::PolymorphicMatcher ArgType(RespExpr::Type t) } MATCHER_P(RespArray, value, "") { - return ExplainMatchResult(testing::AllOf(testing::Field(&RespExpr::type, RespExpr::ARRAY), - testing::Property(&RespExpr::GetVec, value)), - arg, result_listener); + return ExplainMatchResult( + testing::AllOf(ArgType(RespExpr::ARRAY), testing::Property(&RespExpr::GetVec, value)), arg, + result_listener); } inline bool operator==(const RespExpr& left, std::string_view s) { diff --git a/src/server/CMakeLists.txt b/src/server/CMakeLists.txt index 51a0c8fcc..43c753437 100644 --- a/src/server/CMakeLists.txt +++ b/src/server/CMakeLists.txt @@ -62,7 +62,8 @@ endif() find_library(ZSTD_LIB NAMES libzstd.a libzstdstatic.a zstd NAMES_PER_DIR REQUIRED) cxx_link(dfly_transaction dfly_core strings_lib TRDP::fast_float) -cxx_link(dragonfly_lib dfly_transaction dfly_facade redis_lib awsv2_lib strings_lib html_lib +cxx_link(dragonfly_lib dfly_transaction dfly_facade redis_lib awsv2_lib jsonpath + strings_lib html_lib http_client_lib absl::random_random TRDP::jsoncons ${ZSTD_LIB} TRDP::lz4 TRDP::croncpp) diff --git a/src/server/json_family.cc b/src/server/json_family.cc index 4b963c13f..be7762f00 100644 --- a/src/server/json_family.cc +++ b/src/server/json_family.cc @@ -19,7 +19,10 @@ extern "C" { #include #include +#include "base/flags.h" #include "base/logging.h" +#include "core/json/driver.h" +#include "core/json/jsonpath_grammar.hh" #include "core/json_object.h" #include "facade/cmd_arg_parser.h" #include "server/acl/acl_commands_def.h" @@ -30,10 +33,13 @@ extern "C" { #include "server/tiered_storage.h" #include "server/transaction.h" +ABSL_FLAG(bool, jsonpathv2, false, "If true uses Dragonfly jsonpath implementation."); + namespace dfly { using namespace std; using namespace jsoncons; +using facade::kSyntaxErrType; using JsonExpression = jsonpath::jsonpath_expression; using OptBool = optional; @@ -48,6 +54,42 @@ static const char DefaultJsonPath[] = "$"; namespace { +using JsonPathV2 = variant; +using ExprCallback = absl::FunctionRef; + +inline void Evaluate(const JsonExpression& expr, const JsonType& obj, ExprCallback cb) { + expr.evaluate(obj, cb); +} + +inline void Evaluate(const json::Path& expr, const JsonType& obj, ExprCallback cb) { + json::EvaluatePath(expr, obj, [&cb](optional key, const JsonType& val) { + cb(key ? *key : string_view{}, val); + }); +} + +class JsonPathDriver : public json::Driver { + public: + string msg; + void Error(const json::location& l, const std::string& msg) final { + this->msg = absl::StrCat("Error: ", msg); + } +}; + +io::Result JsonPathV2Parse(string_view path) { + if (path.size() > 8_KB) + return nonstd::make_unexpected("Path too long"); + JsonPathDriver driver; + json::Parser parser(&driver); + + driver.SetInput(string(path)); + int res = parser(); + if (res != 0) { + return nonstd::make_unexpected(driver.msg); + } + + return driver.TakePath(); +} + inline OpStatus JsonReplaceVerifyNoOp(JsonType&) { return OpStatus::OK; } @@ -457,7 +499,7 @@ OpResult OpJsonGet(const OpArgs& op_args, string_view key, return out.as(); } -OpResult> OpType(const OpArgs& op_args, string_view key, JsonExpression expression) { +OpResult> OpType(const OpArgs& op_args, string_view key, JsonPathV2 expression) { OpResult result = GetJson(op_args, key); if (!result) { return result.status(); @@ -469,12 +511,11 @@ OpResult> OpType(const OpArgs& op_args, string_view key, JsonExpr vec.emplace_back(JsonTypeToName(val)); }; - expression.evaluate(json_entry, cb); + visit([&](auto&& arg) { Evaluate(arg, json_entry, cb); }, expression); return vec; } -OpResult> OpStrLen(const OpArgs& op_args, string_view key, - JsonExpression expression) { +OpResult> OpStrLen(const OpArgs& op_args, string_view key, JsonPathV2 expression) { OpResult result = GetJson(op_args, key); if (!result) { return result.status(); @@ -489,12 +530,11 @@ OpResult> OpStrLen(const OpArgs& op_args, string_view key, } }; - expression.evaluate(json_entry, cb); + visit([&](auto&& arg) { Evaluate(arg, json_entry, cb); }, expression); return vec; } -OpResult> OpObjLen(const OpArgs& op_args, string_view key, - JsonExpression expression) { +OpResult> OpObjLen(const OpArgs& op_args, string_view key, JsonPathV2 expression) { OpResult result = GetJson(op_args, key); if (!result) { return result.status(); @@ -510,12 +550,11 @@ OpResult> OpObjLen(const OpArgs& op_args, string_view key, } }; - expression.evaluate(json_entry, cb); + visit([&](auto&& arg) { Evaluate(arg, json_entry, cb); }, expression); return vec; } -OpResult> OpArrLen(const OpArgs& op_args, string_view key, - JsonExpression expression) { +OpResult> OpArrLen(const OpArgs& op_args, string_view key, JsonPathV2 expression) { OpResult result = GetJson(op_args, key); if (!result) { return result.status(); @@ -531,7 +570,7 @@ OpResult> OpArrLen(const OpArgs& op_args, string_view key, } }; - expression.evaluate(json_entry, cb); + visit([&](auto&& arg) { Evaluate(arg, json_entry, cb); }, expression); return vec; } @@ -649,7 +688,7 @@ OpResult OpDel(const OpArgs& op_args, string_view key, string_view path) { // Returns a vector of string vectors, // keys within the same object are stored in the same string vector. OpResult> OpObjKeys(const OpArgs& op_args, string_view key, - JsonExpression expression) { + JsonPathV2 expression) { OpResult result = GetJson(op_args, key); if (!result) { return result.status(); @@ -658,6 +697,8 @@ OpResult> OpObjKeys(const OpArgs& op_args, string_view key, vector vec; auto cb = [&vec](const string_view& path, const JsonType& val) { // Aligned with ElastiCache flavor. + DVLOG(2) << "path: " << path << " val: " << val.to_string(); + if (!val.is_object()) { vec.emplace_back(); return; @@ -669,8 +710,8 @@ OpResult> OpObjKeys(const OpArgs& op_args, string_view key, } }; JsonType& json_entry = *(result.value()); + visit([&](auto&& arg) { Evaluate(arg, json_entry, cb); }, expression); - expression.evaluate(json_entry, cb); return vec; } @@ -917,9 +958,8 @@ OpResult> OpArrAppend(const OpArgs& op_args, string_view key, s // Returns a numeric vector representing each JSON value first index of the JSON scalar. // An index value of -1 represents unfound in the array. // JSON scalar has types of string, boolean, null, and number. -OpResult> OpArrIndex(const OpArgs& op_args, string_view key, - JsonExpression expression, const JsonType& search_val, - int start_index, int end_index) { +OpResult> OpArrIndex(const OpArgs& op_args, string_view key, JsonPathV2 expression, + const JsonType& search_val, int start_index, int end_index) { OpResult result = GetJson(op_args, key); if (!result) { return result.status(); @@ -975,12 +1015,12 @@ OpResult> OpArrIndex(const OpArgs& op_args, string_view key, vec.emplace_back(pos); }; JsonType& json_entry = *(result.value()); - expression.evaluate(json_entry, cb); + visit([&](auto&& arg) { Evaluate(arg, json_entry, cb); }, expression); return vec; } // Returns string vector that represents the query result of each supplied key. -vector OpJsonMGet(JsonExpression expression, const Transaction* t, EngineShard* shard) { +vector OpJsonMGet(JsonPathV2 expression, const Transaction* t, EngineShard* shard) { auto args = t->GetShardArgs(shard->shard_id()); DCHECK(!args.empty()); vector response(args.size()); @@ -1002,7 +1042,7 @@ vector OpJsonMGet(JsonExpression expression, const Transaction* t, En }; const JsonType& json_entry = *(json_val); - expression.evaluate(json_entry, cb); + visit([&](auto&& arg) { Evaluate(arg, json_entry, cb); }, expression); if (query_result.empty()) { continue; @@ -1028,8 +1068,7 @@ vector OpJsonMGet(JsonExpression expression, const Transaction* t, En } // Returns numeric vector that represents the number of fields of JSON value at each path. -OpResult> OpFields(const OpArgs& op_args, string_view key, - JsonExpression expression) { +OpResult> OpFields(const OpArgs& op_args, string_view key, JsonPathV2 expression) { OpResult result = GetJson(op_args, key); if (!result) { return result.status(); @@ -1040,13 +1079,12 @@ OpResult> OpFields(const OpArgs& op_args, string_view key, vec.emplace_back(CountJsonFields(val)); }; const JsonType& json_entry = *(result.value()); - expression.evaluate(json_entry, cb); + visit([&](auto&& arg) { Evaluate(arg, json_entry, cb); }, expression); return vec; } // Returns json vector that represents the result of the json query. -OpResult> OpResp(const OpArgs& op_args, string_view key, - JsonExpression expression) { +OpResult> OpResp(const OpArgs& op_args, string_view key, JsonPathV2 expression) { OpResult result = GetJson(op_args, key); if (!result) { return result.status(); @@ -1055,7 +1093,7 @@ OpResult> OpResp(const OpArgs& op_args, string_view key, vector vec; auto cb = [&vec](const string_view& path, const JsonType& val) { vec.emplace_back(val); }; const JsonType& json_entry = *(result.value()); - expression.evaluate(json_entry, cb); + visit([&](auto&& arg) { Evaluate(arg, json_entry, cb); }, expression); return vec; } @@ -1141,6 +1179,17 @@ OpResult OpSet(const OpArgs& op_args, string_view key, string_view path, return operation_result; } +io::Result ParsePathV2(string_view path) { + if (absl::GetFlag(FLAGS_jsonpathv2)) { + return JsonPathV2Parse(path); + } + io::Result expr_result = ParseJsonPath(path); + if (!expr_result) { + return nonstd::make_unexpected(kSyntaxErr); + } + return JsonPathV2(std::move(expr_result.value())); +} + } // namespace // GCC extension of returning a value of multiple statements. The last statement is returned. @@ -1155,6 +1204,16 @@ OpResult OpSet(const OpArgs& op_args, string_view key, string_view path, std::move(*expr_result); \ }) +#define PARSE_PATHV2(path) \ + ({ \ + auto result = ParsePathV2(path); \ + if (!result) { \ + cntx->SendError(result.error()); \ + return; \ + } \ + std::move(*result); \ + }) + void JsonFamily::Set(CmdArgList args, ConnectionContext* cntx) { string_view key = ArgS(args, 0); string_view path = ArgS(args, 1); @@ -1201,7 +1260,7 @@ void JsonFamily::Resp(CmdArgList args, ConnectionContext* cntx) { path = ArgS(args, 1); } - JsonExpression expression = PARSE_PATH_ARG(path); + JsonPathV2 expression = PARSE_PATHV2(path); auto cb = [&](Transaction* t, EngineShard* shard) { return OpResp(t->GetOpArgs(shard), key, std::move(expression)); @@ -1269,7 +1328,7 @@ void JsonFamily::MGet(CmdArgList args, ConnectionContext* cntx) { DCHECK_GE(args.size(), 1U); string_view path = ArgS(args, args.size() - 1); - JsonExpression expression = PARSE_PATH_ARG(path); + JsonPathV2 expression = PARSE_PATHV2(path); Transaction* transaction = cntx->transaction; unsigned shard_count = shard_set->size(); @@ -1319,7 +1378,7 @@ void JsonFamily::ArrIndex(CmdArgList args, ConnectionContext* cntx) { string_view key = ArgS(args, 0); string_view path = ArgS(args, 1); - JsonExpression expression = PARSE_PATH_ARG(path); + JsonPathV2 expression = PARSE_PATHV2(path); optional search_value = JsonFromString(ArgS(args, 2)); if (!search_value) { @@ -1542,7 +1601,7 @@ void JsonFamily::ObjKeys(CmdArgList args, ConnectionContext* cntx) { string_view key = ArgS(args, 0); string_view path = ArgS(args, 1); - JsonExpression expression = PARSE_PATH_ARG(path); + JsonPathV2 expression = PARSE_PATHV2(path); auto cb = [&](Transaction* t, EngineShard* shard) { return OpObjKeys(t->GetOpArgs(shard), key, std::move(expression)); @@ -1653,7 +1712,7 @@ void JsonFamily::Type(CmdArgList args, ConnectionContext* cntx) { string_view key = ArgS(args, 0); string_view path = ArgS(args, 1); - JsonExpression expression = PARSE_PATH_ARG(path); + JsonPathV2 expression = PARSE_PATHV2(path); auto cb = [&](Transaction* t, EngineShard* shard) { return OpType(t->GetOpArgs(shard), key, std::move(expression)); @@ -1682,7 +1741,7 @@ void JsonFamily::ArrLen(CmdArgList args, ConnectionContext* cntx) { string_view key = ArgS(args, 0); string_view path = ArgS(args, 1); - JsonExpression expression = PARSE_PATH_ARG(path); + JsonPathV2 expression = PARSE_PATHV2(path); auto cb = [&](Transaction* t, EngineShard* shard) { return OpArrLen(t->GetOpArgs(shard), key, std::move(expression)); @@ -1702,7 +1761,7 @@ void JsonFamily::ObjLen(CmdArgList args, ConnectionContext* cntx) { string_view key = ArgS(args, 0); string_view path = ArgS(args, 1); - JsonExpression expression = PARSE_PATH_ARG(path); + JsonPathV2 expression = PARSE_PATHV2(path); auto cb = [&](Transaction* t, EngineShard* shard) { return OpObjLen(t->GetOpArgs(shard), key, std::move(expression)); @@ -1722,7 +1781,7 @@ void JsonFamily::StrLen(CmdArgList args, ConnectionContext* cntx) { string_view key = ArgS(args, 0); string_view path = ArgS(args, 1); - JsonExpression expression = PARSE_PATH_ARG(path); + JsonPathV2 expression = PARSE_PATHV2(path); auto cb = [&](Transaction* t, EngineShard* shard) { return OpStrLen(t->GetOpArgs(shard), key, std::move(expression)); diff --git a/src/server/json_family_test.cc b/src/server/json_family_test.cc index 33fd3aaac..856e65970 100644 --- a/src/server/json_family_test.cc +++ b/src/server/json_family_test.cc @@ -165,9 +165,8 @@ TEST_F(JsonFamilyTest, Type) { ASSERT_THAT(resp, "OK"); resp = Run({"JSON.TYPE", "json", "$[*]"}); - ASSERT_EQ(RespExpr::ARRAY, resp.type); - EXPECT_THAT(resp.GetVec(), - ElementsAre("integer", "number", "string", "boolean", "null", "object", "array")); + ASSERT_THAT(resp, RespArray(ElementsAre("integer", "number", "string", "boolean", "null", + "object", "array"))); resp = Run({"JSON.TYPE", "json", "$[10]"}); EXPECT_THAT(resp, ArgType(RespExpr::NIL)); @@ -589,7 +588,7 @@ TEST_F(JsonFamilyTest, ObjKeys) { ASSERT_THAT(resp, ArrLen(2)); const auto& arr1 = resp.GetVec(); EXPECT_THAT(arr1[0], ArgType(RespExpr::NIL_ARRAY)); - EXPECT_THAT(arr1[1].GetVec(), ElementsAre("b", "c")); + EXPECT_THAT(arr1[1], RespArray(ElementsAre("b", "c"))); json = R"( {"a":{}, "b":{"c":{"d": {"e": 1337}}}}