How to export logs in foundry fuzz tests?

Question

I am trying to visualise how often each (logic) branch of the fuzz code is reached. I did not know how to keep a tracker inside a forge .sol test file that undergoes multiple fuzz test calls (because each fuzz run of a single tests resets the:

  uint256 private _validInitialisationCounter;
  uint256 private _testRunsCounter;

Values within the fuzz test contract:

contract FuzzDebug is PRBTest, StdCheats, IFuzzDebug {
  address internal _projectLead;
  TestInitialisationHelper private _testInitialisationHelper;
  Helper private _helper;
  uint256 private _validInitialisationCounter;
  uint256 private _testRunsCounter;
...
}

Then I found out I can read and write to json files from the .sol test files in foundry. So at the start of the test, I can load the counters from the branch hits from the .json file, and increment them during the fuzz run, and at the end of the run export them again, storing and incrementing them over multiple fuzz runs.

However, that seems like a rather convoluted method to track the branch hit rate of specific fuzz test branches. Hence, I would like to ask:

Is there a more simple way to store cumulative emit Log() events to file somewhere per file per fuzz test over multiple fuzz runs?

Answer 1

Description

It appears to me that at the time of writing, this functionality is not yet supported, based on these discussions:

https://github.com/foundry-rs/foundry/issues/2552 https://github.com/crytic/medusa/issues/234 https://github.com/foundry-rs/foundry/pull/7899 https://github.com/foundry-rs/foundry/issues/4300

So I built a hacky way around logging Fuzz runs. It leverages:

• setUp() is ran once before each test.
• fuzzRuns are ran multiple times per test.
• A temporary file has a "last modified" date that can be read from test.

And I use the following terminology:

• A test run can run multiple test.
• A test can run multiple Fuzz runs of that same test.

So in the setUp() I delete a temporary file if it exists, and in the actual test, I create the temp file if it does not exist yet. This means that the first test of each test run, creates a new "last modified" timestamp. This "last modified" timestamp is then used to store a file per test run per test. This file contains the data you wish to track.

For me, it was important to verify that each "branch" of the test logic is hit by the fuzz test. Because it could be that the randomness avoids some test cases, which could mean some behavior would not be tested even though the test passes, giving a false sense of confidence.

Code

The solution consist of 3 files in the test/ directory:

TestFileLogging.sol

pragma solidity >=0.8.26 <0.9.0;

import "forge-std/src/Vm.sol" as vm;
import { PRBTest } from "@prb/test/src/PRBTest.sol";
import { StdCheats } from "forge-std/src/StdCheats.sol";

import "@openzeppelin/contracts/utils/Strings.sol";

error LogFileNotCreated(string message, string fileName);
error SomeFileDoesNotExist(string message, string fileName);
error SomeFileNotCreated(string message, string fileName);

interface ITestFileLogging {
  // solhint-disable-next-line foundry-test-functions
  function createFileIfNotExists(
    string memory serialisedTextString,
    string memory filePath
  ) external returns (uint256 lastModified);

  // solhint-disable-next-line foundry-test-functions
  function overwriteFileContent(string memory serialisedTextString, string memory filePath) external;

  // solhint-disable-next-line foundry-test-functions
  function createLogFileIfItDoesNotExist(
    string memory tempFileName,
    string memory serialisedTextString
  ) external returns (string memory hitRateFilePath);

  // solhint-disable-next-line foundry-test-functions
  function readDataFromFile(string memory path) external view returns (bytes memory data);
}

contract TestFileLogging is PRBTest, StdCheats, ITestFileLogging {
  // solhint-disable-next-line foundry-test-functions
  function createFileIfNotExists(
    string memory serialisedTextString,
    string memory filePath
  ) public override returns (uint256 lastModified) {
    if (!vm.isFile(filePath)) {
      overwriteFileContent(serialisedTextString, filePath);
    }
    if (!vm.isFile(filePath)) {
      revert SomeFileNotCreated("Some file not created.", filePath);
    }
    return vm.fsMetadata(filePath).modified;
  }

  // solhint-disable-next-line foundry-test-functions
  function overwriteFileContent(string memory serialisedTextString, string memory filePath) public override {
    vm.writeJson(serialisedTextString, filePath);
    if (!vm.isFile(filePath)) {
      revert SomeFileDoesNotExist("Some file does not exist.", filePath);
    }
  }

  // solhint-disable-next-line foundry-test-functions
  function createLogFileIfItDoesNotExist(
    string memory tempFileName,
    string memory serialisedTextString
  ) public override returns (string memory hitRateFilePath) {
    // Specify the logging directory and filepath.
    uint256 timeStamp = createFileIfNotExists(serialisedTextString, tempFileName);
    string memory logDir = string(abi.encodePacked("test_logging/", Strings.toString(timeStamp)));
    hitRateFilePath = string(abi.encodePacked(logDir, "/DebugTest.txt"));

    // If the log file does not yet exist, create it.
    if (!vm.isFile(hitRateFilePath)) {
      // Create logging structure
      vm.createDir(logDir, true);
      overwriteFileContent(serialisedTextString, hitRateFilePath);

      // Assort logging file exists.
      if (!vm.isFile(hitRateFilePath)) {
        revert LogFileNotCreated("LogFile not created.", hitRateFilePath);
      }
    }
    return hitRateFilePath;
  }

  // solhint-disable-next-line foundry-test-functions
  function readDataFromFile(string memory path) public view override returns (bytes memory data) {
    string memory fileContent = vm.readFile(path);
    data = vm.parseJson(fileContent);
    return data;
  }
}

TestConstants.sol

// SPDX-License-Identifier: UNLICENSED
pragma solidity >=0.8.26 <0.9.0;
string constant _LOG_TIME_CREATOR = "temp.txt"; // File used to create the timestamp to log the fuzz tests.

FuzzTest.sol

// SPDX-License-Identifier: UNLICENSED
pragma solidity >=0.8.26 <0.9.0;

import "forge-std/src/Vm.sol" as vm;
import { PRBTest } from "@prb/test/src/PRBTest.sol";
import { StdCheats } from "forge-std/src/StdCheats.sol";

import "@openzeppelin/contracts/utils/Strings.sol";

import "test/TestConstants.sol";
import { TestFileLogging } from "../TestFileLogging.sol";

struct HitRatesReturnAll {
  uint256 largeValue;
  uint256 smallValue;
}

interface IFuzzTest {
  function setUp() external;

  // solhint-disable-next-line foundry-test-functions
  function converthitRatesToString(
    HitRatesReturnAll memory hitRates
  ) external returns (string memory serialisedTextString);

  // solhint-disable-next-line foundry-test-functions
  function updateLogFile() external returns (string memory hitRateFilePath, HitRatesReturnAll memory hitRates);

  function testFuzzDebug(uint256 randomValue) external;

  // solhint-disable-next-line foundry-test-functions
  function initialiseHitRates() external pure returns (HitRatesReturnAll memory hitRates);
}

contract FuzzTest is PRBTest, StdCheats, IFuzzTest {
  address internal _projectLead;
  TestFileLogging private _testFileLogging;

  // solhint-disable-next-line foundry-test-functions
  function converthitRatesToString(
    HitRatesReturnAll memory hitRates
  ) public override returns (string memory serialisedTextString) {
    string memory obj1 = "ThisValueDissapearsIntoTheVoid";
    vm.serializeUint(obj1, "largeValue", hitRates.largeValue);
    serialisedTextString = vm.serializeUint(obj1, "smallValue", hitRates.smallValue);
    return serialisedTextString;
  }

  // solhint-disable-next-line foundry-test-functions
  function updateLogFile() public override returns (string memory hitRateFilePath, HitRatesReturnAll memory hitRates) {
    // solhint-disable-next-line foundry-test-functions
    hitRates = initialiseHitRates();
    // Output hit rates to file if they do not exist yet.
    // solhint-disable-next-line foundry-test-functions
    string memory serialisedTextString = converthitRatesToString(hitRates);
    hitRateFilePath = _testFileLogging.createLogFileIfItDoesNotExist(_LOG_TIME_CREATOR, serialisedTextString);
    // Read the latest hitRates from file.
    bytes memory data = _testFileLogging.readDataFromFile(hitRateFilePath);
    hitRates = abi.decode(data, (HitRatesReturnAll));

    return (hitRateFilePath, hitRates);
  }

  function setUp() public virtual override {
    _testFileLogging = new TestFileLogging();
    // Delete the temp file if it already exists.
    if (vm.isFile(_LOG_TIME_CREATOR)) {
      vm.removeFile(_LOG_TIME_CREATOR);
    }
  }

  /**
  @dev The investor has invested 0.5 eth, and the investment target is 0.6 eth after 12 weeks.
  So the investment target is not reached, so all the funds should be returned.
   */
  function testFuzzDebug(uint256 randomValue) public virtual override {
    // solhint-disable-next-line foundry-test-functions
    (string memory hitRateFilePath, HitRatesReturnAll memory hitRates) = updateLogFile();

    if (randomValue > 42) {
      ++hitRates.largeValue;
    } else {
      ++hitRates.smallValue;
    }
    emit Log("Outputting File");
    string memory serialisedTextString = converthitRatesToString(hitRates);
    _testFileLogging.overwriteFileContent(serialisedTextString, hitRateFilePath);
    emit Log("Outputted File");
  }

  // solhint-disable-next-line foundry-test-functions
  function initialiseHitRates() public pure override returns (HitRatesReturnAll memory hitRates) {
    return HitRatesReturnAll({ largeValue: 0, smallValue: 0 });
  }
}

Example

And here is a clonable MWE repo that implements it. Disclaimer: I am involved in writing that repo.