diff --git a/CHANGELOG.md b/CHANGELOG.md index 03eba64d8..1c4fbe494 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,12 @@ This is a list of notable changes to Hyperscan, in reverse chronological order. +## [4.4.1] 2017-02-28 +- Bugfixes to fix issues where stale data was being referenced in scratch + memory. In particular this may have resulted in hs_close_stream() + referencing data from other previously scanned streams. This may result in + incorrect matches being been reported. + ## [4.4.0] 2017-01-20 - Introduce the "fat runtime" build. This will build several variants of the Hyperscan scanning engine specialised for different processor feature sets, @@ -136,7 +142,9 @@ This is a list of notable changes to Hyperscan, in reverse chronological order. supplied with a NULL scratch pointer if no matches are required. This is in line with the behaviour of `hs_close_stream()`. - Disallow bounded repeats with a very large minimum repeat but no maximum, - i.e. {N,} for very large N. + i.e. { + N, +} for very large N. - Reduce compile memory usage in literal set explansion for some large cases. ## [4.0.0] 2015-10-20 diff --git a/CMakeLists.txt b/CMakeLists.txt index 3a7d40ea8..7ede52b45 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -3,7 +3,7 @@ project (Hyperscan C CXX) set (HS_MAJOR_VERSION 4) set (HS_MINOR_VERSION 4) -set (HS_PATCH_VERSION 0) +set (HS_PATCH_VERSION 1) set (HS_VERSION ${HS_MAJOR_VERSION}.${HS_MINOR_VERSION}.${HS_PATCH_VERSION}) set(CMAKE_MODULE_PATH ${PROJECT_SOURCE_DIR}/cmake) diff --git a/src/nfa/lbr.c b/src/nfa/lbr.c index 3075be333..d403733a6 100644 --- a/src/nfa/lbr.c +++ b/src/nfa/lbr.c @@ -1,5 +1,5 @@ /* - * Copyright (c) 2015-2016, Intel Corporation + * Copyright (c) 2015-2017, Intel Corporation * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: @@ -77,6 +77,7 @@ void lbrExpandState(const struct lbr_common *l, u64a offset, const struct RepeatInfo *info = getRepeatInfo(l); repeatUnpack(stream_state, info, offset, &lstate->ctrl); + lstate->lastEscape = 0; } static really_inline diff --git a/src/nfa/limex_compile.cpp b/src/nfa/limex_compile.cpp index ba4d0f0d9..c75eae597 100644 --- a/src/nfa/limex_compile.cpp +++ b/src/nfa/limex_compile.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2015-2016, Intel Corporation + * Copyright (c) 2015-2017, Intel Corporation * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: @@ -1803,6 +1803,12 @@ struct Factory { assert(cyclic != NO_STATE); maskSetBit(limex->repeatCyclicMask, cyclic); } + /* also include tugs in repeat cyclic mask */ + for (NFAVertex v : args.tugs) { + u32 v_state = args.state_ids.at(v); + assert(v_state != NO_STATE); + maskSetBit(limex->repeatCyclicMask, v_state); + } } static diff --git a/src/nfa/limex_internal.h b/src/nfa/limex_internal.h index 723803c19..ccbf34223 100644 --- a/src/nfa/limex_internal.h +++ b/src/nfa/limex_internal.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2015-2016, Intel Corporation + * Copyright (c) 2015-2017, Intel Corporation * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: @@ -151,7 +151,7 @@ struct LimExNFA##size { \ * followers */ \ u_##size compressMask; /**< switch off before compress */ \ u_##size exceptionMask; \ - u_##size repeatCyclicMask; \ + u_##size repeatCyclicMask; /**< also includes tug states */ \ u_##size zombieMask; /**< zombie if in any of the set states */ \ u_##size shift[MAX_SHIFT_COUNT]; \ u32 shiftCount; /**< number of shift masks used */ \ diff --git a/src/nfa/limex_runtime_impl.h b/src/nfa/limex_runtime_impl.h index 45ceb2b5e..016d1f924 100644 --- a/src/nfa/limex_runtime_impl.h +++ b/src/nfa/limex_runtime_impl.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2015-2016, Intel Corporation + * Copyright (c) 2015-2017, Intel Corporation * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: @@ -393,19 +393,16 @@ void COMPRESS_REPEATS_FN(const IMPL_NFA_T *limex, void *dest, void *src, DEBUG_PRINTF("repeat %u\n", i); const struct NFARepeatInfo *info = GET_NFA_REPEAT_INFO_FN(limex, i); - if (!TESTBIT_STATE(s, info->cyclicState)) { + const ENG_STATE_T *tug_mask = + (const ENG_STATE_T *)((const char *)info + info->tugMaskOffset); + /* repeat may still be inspected if its tug state is on */ + if (!TESTBIT_STATE(s, info->cyclicState) + && ISZERO_STATE(AND_STATE(s, LOAD_FROM_ENG(tug_mask)))) { DEBUG_PRINTF("is dead\n"); continue; } const struct RepeatInfo *repeat = getRepeatInfo(info); - if (repeatHasMatch(repeat, &ctrl[i], state_base + info->stateOffset, - offset) == REPEAT_STALE) { - DEBUG_PRINTF("is stale, clearing state\n"); - CLEARBIT_STATE(&s, info->cyclicState); - continue; - } - DEBUG_PRINTF("packing state (packedCtrlOffset=%u)\n", info->packedCtrlOffset); repeatPack(state_base + info->packedCtrlOffset, repeat, &ctrl[i], @@ -448,8 +445,11 @@ void EXPAND_REPEATS_FN(const IMPL_NFA_T *limex, void *dest, const void *src, for (u32 i = 0; i < limex->repeatCount; i++) { DEBUG_PRINTF("repeat %u\n", i); const struct NFARepeatInfo *info = GET_NFA_REPEAT_INFO_FN(limex, i); + const ENG_STATE_T *tug_mask = + (const ENG_STATE_T *)((const char *)info + info->tugMaskOffset); - if (!TESTBIT_STATE(cyclics, info->cyclicState)) { + if (!TESTBIT_STATE(cyclics, info->cyclicState) + && ISZERO_STATE(AND_STATE(cyclics, LOAD_FROM_ENG(tug_mask)))) { DEBUG_PRINTF("is dead\n"); continue; } diff --git a/src/nfa/repeat.c b/src/nfa/repeat.c index 339829a52..5b2e4df4e 100644 --- a/src/nfa/repeat.c +++ b/src/nfa/repeat.c @@ -1,5 +1,5 @@ /* - * Copyright (c) 2015-2016, Intel Corporation + * Copyright (c) 2015-2017, Intel Corporation * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: @@ -177,6 +177,10 @@ u64a repeatLastTopRange(const union RepeatControl *ctrl, const void *state) { u64a repeatLastTopBitmap(const union RepeatControl *ctrl) { const struct RepeatBitmapControl *xs = &ctrl->bitmap; + if (!xs->bitmap) { + /* last top was too long ago */ + return 0; + } return xs->offset + 63 - clz64(xs->bitmap); } diff --git a/src/rose/program_runtime.h b/src/rose/program_runtime.h index 5b2c829f3..e883c239e 100644 --- a/src/rose/program_runtime.h +++ b/src/rose/program_runtime.h @@ -1209,6 +1209,8 @@ hwlmcb_rv_t roseEnginesEod(const struct RoseEngine *rose, const u8 *aa = getActiveLeafArray(rose, scratch->core_info.state); const u32 aaCount = rose->activeArrayCount; + const u32 qCount = rose->queueCount; + struct fatbit *aqa = scratch->aqa; const struct mmbit_sparse_iter *it = getByOffset(rose, iter_offset); assert(ISALIGNED(it)); @@ -1221,6 +1223,10 @@ hwlmcb_rv_t roseEnginesEod(const struct RoseEngine *rose, qi = mmbit_sparse_iter_next(aa, aaCount, qi, &idx, it, si_state)) { DEBUG_PRINTF("checking nfa %u\n", qi); struct mq *q = scratch->queues + qi; + if (!fatbit_set(aqa, qCount, qi)) { + initQueue(q, qi, rose, scratch); + } + assert(q->nfa == getNfaByQueue(rose, qi)); assert(nfaAcceptsEod(q->nfa)); diff --git a/src/runtime.c b/src/runtime.c index 88e866dc8..a2ed10260 100644 --- a/src/runtime.c +++ b/src/runtime.c @@ -186,6 +186,18 @@ void rawBlockExec(const struct RoseEngine *rose, struct hs_scratch *scratch) { roseBlockExec(rose, scratch); } +static really_inline +void pureLiteralInitScratch(struct hs_scratch *scratch, u64a offset) { + // Some init has already been done. + assert(offset == scratch->core_info.buf_offset); + + scratch->tctxt.lit_offset_adjust = offset + 1; + scratch->tctxt.lastEndOffset = offset; + scratch->tctxt.delayLastEndOffset = offset; + scratch->tctxt.filledDelayedSlots = 0; + scratch->al_log_sum = 0; +} + static really_inline void pureLiteralBlockExec(const struct RoseEngine *rose, struct hs_scratch *scratch) { @@ -198,9 +210,8 @@ void pureLiteralBlockExec(const struct RoseEngine *rose, size_t length = scratch->core_info.len; DEBUG_PRINTF("rose engine %d\n", rose->runtimeImpl); - // RoseContext values that need to be set for use by roseCallback. + pureLiteralInitScratch(scratch, 0); scratch->tctxt.groups = rose->initialGroups; - scratch->tctxt.lit_offset_adjust = 1; hwlmExec(ftable, buffer, length, 0, roseCallback, scratch, rose->initialGroups); @@ -743,9 +754,8 @@ void pureLiteralStreamExec(struct hs_stream *stream_state, DEBUG_PRINTF("::: streaming rose ::: offset = %llu len = %zu\n", stream_state->offset, scratch->core_info.len); - // RoseContext values that need to be set for use by roseCallback. + pureLiteralInitScratch(scratch, stream_state->offset); scratch->tctxt.groups = loadGroups(rose, scratch->core_info.state); - scratch->tctxt.lit_offset_adjust = scratch->core_info.buf_offset + 1; // Pure literal cases don't have floatingMinDistance set, so we always // start the match region at zero. diff --git a/unit/hyperscan/behaviour.cpp b/unit/hyperscan/behaviour.cpp index a816969c5..f15e71716 100644 --- a/unit/hyperscan/behaviour.cpp +++ b/unit/hyperscan/behaviour.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2015-2016, Intel Corporation + * Copyright (c) 2015-2017, Intel Corporation * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: @@ -1172,6 +1172,106 @@ TEST(HyperscanTestBehaviour, Vectored7) { hs_free_database(db); } +TEST(HyperscanTestBehaviour, MultiStream1) { + hs_error_t err; + + // build a database + hs_database_t *db = buildDB("foo.*bar.*\\b", 0, 0, HS_MODE_STREAM); + ASSERT_TRUE(db != nullptr); + + // alloc some scratch + hs_scratch_t *scratch = nullptr; + err = hs_alloc_scratch(db, &scratch); + ASSERT_EQ(HS_SUCCESS, err); + EXPECT_TRUE(scratch != nullptr); + + hs_stream_t *stream = nullptr; + err = hs_open_stream(db, 0, &stream); + ASSERT_EQ(HS_SUCCESS, err); + ASSERT_TRUE(stream != nullptr); + + hs_stream_t *stream2 = nullptr; + err = hs_open_stream(db, 0, &stream2); + ASSERT_EQ(HS_SUCCESS, err); + ASSERT_TRUE(stream2 != nullptr); + + matchCount = 0; + const string data("foo bara"); + err = hs_scan_stream(stream, data.c_str(), data.size(), 0, scratch, + countHandler, nullptr); + ASSERT_EQ(HS_SUCCESS, err); + EXPECT_EQ(0U, matchCount); // hasn't matched until stream end + + const string data2("foo bar "); + err = hs_scan_stream(stream2, data2.c_str(), data2.size(), 0, scratch, + nullptr, nullptr); + ASSERT_EQ(HS_SUCCESS, err); + EXPECT_EQ(0U, matchCount); + + err = hs_close_stream(stream, scratch, countHandler, nullptr); + ASSERT_EQ(HS_SUCCESS, err); + EXPECT_EQ(1U, matchCount); + + err = hs_close_stream(stream2, scratch, countHandler, nullptr); + ASSERT_EQ(HS_SUCCESS, err); + EXPECT_EQ(1U, matchCount); + + // teardown + err = hs_free_scratch(scratch); + ASSERT_EQ(HS_SUCCESS, err); + hs_free_database(db); +} + +TEST(HyperscanTestBehaviour, MultiStream2) { + hs_error_t err; + + // build a database + hs_database_t *db = buildDB("foo.*bar.*\\b", 0, 0, HS_MODE_STREAM); + ASSERT_TRUE(db != nullptr); + + // alloc some scratch + hs_scratch_t *scratch = nullptr; + err = hs_alloc_scratch(db, &scratch); + ASSERT_EQ(HS_SUCCESS, err); + EXPECT_TRUE(scratch != nullptr); + + hs_stream_t *stream = nullptr; + err = hs_open_stream(db, 0, &stream); + ASSERT_EQ(HS_SUCCESS, err); + ASSERT_TRUE(stream != nullptr); + + hs_stream_t *stream2 = nullptr; + err = hs_open_stream(db, 0, &stream2); + ASSERT_EQ(HS_SUCCESS, err); + ASSERT_TRUE(stream2 != nullptr); + + matchCount = 0; + const string data2("foo bar "); + err = hs_scan_stream(stream2, data2.c_str(), data2.size(), 0, scratch, + nullptr, nullptr); + ASSERT_EQ(HS_SUCCESS, err); + EXPECT_EQ(0U, matchCount); + + const string data("foo bara"); + err = hs_scan_stream(stream, data.c_str(), data.size(), 0, scratch, + countHandler, nullptr); + ASSERT_EQ(HS_SUCCESS, err); + EXPECT_EQ(0U, matchCount); // hasn't matched until stream end + + err = hs_close_stream(stream2, scratch, countHandler, nullptr); + ASSERT_EQ(HS_SUCCESS, err); + EXPECT_EQ(0U, matchCount); + + err = hs_close_stream(stream, scratch, countHandler, nullptr); + ASSERT_EQ(HS_SUCCESS, err); + EXPECT_EQ(1U, matchCount); + + // teardown + err = hs_free_scratch(scratch); + ASSERT_EQ(HS_SUCCESS, err); + hs_free_database(db); +} + TEST(regression, UE_1005) { hs_error_t err; vector patterns;