Test: Cover malformed and truncated tx encodings #36456

suggestion that reads more clear to me personally, and:

1. Replace truncation checks with a prefix loop
2. Removes DecodeHexTx checks. DecodeHexTx in general doesn't promise to reject all mutations or truncations

  diff --git a/src/test/transaction_tests.cpp b/src/test/transaction_tests.cpp
  --- a/src/test/transaction_tests.cpp
  +++ b/src/test/transaction_tests.cpp
  @@ -1398,31 +1398,24 @@ BOOST_AUTO_TEST_CASE(tx_malformed_serialization)
   {
  -    // A transaction with one input, one output and a one-element witness,
  split into its fields.
  -    const std::string version{"01000000"};
  -    const std::string marker_flag{"0001"};
  -    const std::string inputs{"01" + std::string(64, '0') + "00000000" "00"
  "ffffffff"};
  -    const std::string outputs{"01" "0100000000000000" "0151"};
  -    const std::string witness{"01" "01aa"};
  -    const std::string locktime{"00000000"};
  -
  +    // A transaction with one input, one output and a one-element witness.
       CMutableTransaction mtx;
  -    BOOST_REQUIRE(DecodeHexTx(mtx, version + marker_flag + inputs + outputs +
  witness + locktime));
  -    BOOST_CHECK_EQUAL(mtx.vin.at(0).scriptWitness.stack.size(), 1U);
  +    mtx.vin.resize(1);
  +    mtx.vin[0].scriptWitness.stack = {{0xaa}};
  +    mtx.vout.emplace_back(1, CScript{} << OP_TRUE);
  +    DataStream ser;
  +    ser << TX_WITH_WITNESS(mtx);

  -    // Each malformed serialization must fail to deserialize, and DecodeHexTx
  must
  -    // not fall back to reading the same bytes as some other transaction
  without
  -    // witness data.
  -    const auto check_malformed{[](const std::string& hex, const std::string&
  reason) {
  -        CMutableTransaction tx;
  -        BOOST_CHECK_EXCEPTION(DataStream{ParseHex(hex)} >>
  TX_WITH_WITNESS(tx), std::ios_base::failure, HasReason{reason});
  -        BOOST_CHECK(!DecodeHexTx(tx, hex, /*try_no_witness=*/true,
  /*try_witness=*/true));
  -    }};
  +    CMutableTransaction tx;
  +    DataStream{std::span{ser}} >> TX_WITH_WITNESS(tx);
  +    BOOST_CHECK(CTransaction{tx}.HasWitness());
  +
  +    // Every strict prefix ends inside some field, such as a CompactSize, an 
  output
  +    // script, a witness element or the locktime, and fails to deserialize.
  +    for (size_t len{0}; len < ser.size(); ++len) {
  +        BOOST_CHECK_EXCEPTION(DataStream{std::span{ser}.first(len)} >> 
  TX_WITH_WITNESS(tx), std::ios_base::failure, HasReason{"end of data"});
  +    }

       // A flag other than 1 is unknown, including one without the witness bit.
  -    check_malformed(version + "0002" + inputs + outputs + witness + locktime,
  "Unknown transaction optional data");
  -    // A witness element that is longer than the remaining data.
  -    check_malformed(version + marker_flag + inputs + outputs + "01" "06aa" + 
  locktime, "end of data");
  -    // A truncated locktime.
  -    check_malformed(version + marker_flag + inputs + outputs + witness + 
  "000000", "end of data");
  -    // An output script that is truncated after its length.
  -    check_malformed(version + marker_flag + inputs + "01" "0100000000000000" 
  "01", "end of data");
  +    DataStream bad_flag;
  +    bad_flag << mtx.version << uint8_t{0x00} << uint8_t{0x02} << mtx.vin << 
  mtx.vout << mtx.vin[0].scriptWitness.stack << mtx.nLockTime;
  +    BOOST_CHECK_EXCEPTION(bad_flag >> TX_WITH_WITNESS(tx), 
  std::ios_base::failure, HasReason{"Unknown transaction optional data"});
   }

done

Missed opportunity not to also do too large: "ReadCompactSize(): size too large" doesn't have any coverage.