What The Fuzz 분석

What The Fuzz 분석

What The Fuzz

wtf.cc 에서 모든 인수에 대해서 CLI를 이용하여 파싱한 다음, Master(=서버)/Fuzz(=클라이언트) 로 나누어짐

중점적으로 볼 부분

• Server - 어떻게 Mutation 하여 testcase를 생성하는가
• Client - 어떻게 실행시키는가
• Client - 어떻게 Coverage를 측정하는가

Master (=Server) Node

Fuzz 노드의 경우 backend 부분에 대한 초기화가 추가로 이루어짐

 CLI11_PARSE(Wtf, argc, argv);

  //
  // If we are in master mode, no need to initialize the heavy machinery.
  //

  // 마스터 노드 = 서버 노드인 경우 바로 리턴함
  // 이 후의 과정은 Fuzz 노드 = 클라이언트 노드의 backend 초기화 과정임
  if (Wtf.got_subcommand("master")) {
    return MasterSubcommand(Opts);
  }

MasterSubcommand는 Server_t.Run()을 호출함

int MasterSubcommand(const Options_t &Opts) {
  return Server_t(Opts.Master).Run();
}

Server_t 객체는 소캣의 read/write, coverage, corpus, random, mutation 방법 등의 정보를 갖는 서버 객체라고 할 수 있음.

class ServerStats_t {
  //
  // If we're asked to print stats and it hasn't been at least 10 seconds since
  // the last refresh we won't display them.
  //
  constexpr static uint64_t RefreshRate = 10;
  //
  // The number of files in the corpus.
  //
  uint64_t CorpusSize_ = 0;
  //
  // The aggregated size of the corpus in bytes.
  //
  uint64_t CorpusBytes_ = 0;
  //
  // The amount of coverage hit.
  //
  uint64_t Coverage_ = 0;
  //
  // The last coverage displayed on the output (used to compute the difference
  // since the previous time the status changed).
  //
  uint64_t LastCoverage_ = 0;
  //
  // The number of testcase the nodes have executed.
  //
  uint64_t TestcasesNumber_ = 0;
  //
  // The number of currently connected clients.
  //
  uint64_t Clients_ = 0;
  //
  // The number of testcases that crashed.
  //
  uint64_t Crashes_ = 0;
  //
  // The number of testcases that triggered a cr3 change.
  //
  uint64_t Cr3s_ = 0;
  //
  // The number of testcases that triggered a timeout.
  //
  uint64_t Timeouts_ = 0;
  //
  // This is a point in time when the server stats are created.
  //
  chrono::system_clock::time_point Start_ = chrono::system_clock::now();
  //
  // This is a point in time when the first client connects to the server.
  //
  chrono::system_clock::time_point FirstClientStart_;
  //
  // The first time a client connects.
  //
  bool FirstClient_ = true;
  //
  // This is the last time we printed the stats.
  //
  chrono::system_clock::time_point LastPrint_ = chrono::system_clock::now();
  //
  // This is the last time we received a new coverage.
  //
  chrono::system_clock::time_point LastCov_ = chrono::system_clock::now();

생성자는 다음과 같음.Opts_, Rng_, Corpus_

ReadSet_.fds_bits 배열을 0으로 초기화함.

explicit Server_t(const MasterOptions_t &Opts)
      : Opts_(Opts), Rng_(Opts.Seed), Corpus_(Opts.OutputsPath, Rng_) {
    FD_ZERO(&ReadSet_);
    FD_ZERO(&WriteSet_);
  }

Server_t->Run()

int Run() {

    //
    // Set up RNG.
    //

    fmt::print("Seeded with {}\n", Opts_.Seed);

    //
    // Initialize our internal state.
    //

    ScratchBufferGrip_ = std::make_unique<uint8_t[]>(_1MB);
    ScratchBuffer_ = {ScratchBufferGrip_.get(), _1MB};

    if (Opts_.TestcaseBufferMaxSize > ScratchBuffer_.size_bytes()) {
      fmt::print("The biggest testcase would not fit in the scratch buffer\n");
      return EXIT_FAILURE;
    }

    //
    // We'll have at most FD_SETSIZE clients, so let's preallocate whatever we
    // can.
    //

    std::vector<SocketFd_t> ReadFds, WriteFds;
    ReadFds.reserve(FD_SETSIZE);
    WriteFds.reserve(FD_SETSIZE);
    Clients_.reserve(FD_SETSIZE);

    //
    // Instantiate the mutator.
    //

    Mutator_ = std::make_unique<LibfuzzerMutator_t>(Rng_);

    //
    // Prepare initial seeds.
    //

    fmt::print("Iterating through the corpus..\n");
    const fs::directory_iterator DirIt(Opts_.InputsPath);
    for (const auto &DirEntry : DirIt) {
      Paths_.emplace_back(DirEntry);
    }

    //
    // Note that we use rbegin / rend because we want to order the vector from
    // biggest to smallest. We do that because that it's easy to pop data off
    // the vector and it means we don't have to move items either.
    //

    fmt::print("Sorting through the {} entries..\n", Paths_.size());
    std::sort(Paths_.rbegin(), Paths_.rend(), CompareTwoFileBySize);

    //
    // Let's kick off the server.
    //

    fmt::print("Running server on {}..\n", Opts_.Address);
    auto ServerOpt = Listen(Opts_.Address);
    if (!ServerOpt) {
      fmt::print("Listen() failed\n");
      return EXIT_FAILURE;
    }

    //
    // The server socket is also a client.
    //

    Server_ = *ServerOpt;
    Clients_.emplace(Server_, SocketState_t::Read);

    int Ret = EXIT_SUCCESS;
    while (Ret == EXIT_SUCCESS) {

      //
      // Zero out the sets.
      //

      FD_ZERO(&ReadSet_);
      FD_ZERO(&WriteSet_);
      ReadFds.clear();
      WriteFds.clear();

      //
      // For now, the Server socket is the biggest fd AFAWK.
      //

      SocketFd_t MaxFd = Server_;

      //
      // Let's iterate through our fds and put them in the right vector based on
      // their states. While doing that, also let's keep looking for the biggest
      // fd.
      // The reason for having ReadFds / WriteFds vectors are:
      //   1/ I don't think there's a cross-platform way of iterating through
      //   the fdset_t internal data structure to find the fds,
      //   2/ Instead, if iterate over Clients_, accepting / disconnecting
      //   clients modifies Clients_ *while* getting iterated on which is
      //   annoying. Instead, we keep those two small vectors arround.
      //

      for (const auto &[Fd, State] : Clients_) {
        MaxFd = std::max(MaxFd, Fd);
        if (State == SocketState_t::Read) {
          FD_SET(Fd, &ReadSet_);
          ReadFds.emplace_back(Fd);
        } else {
          FD_SET(Fd, &WriteSet_);
          WriteFds.emplace_back(Fd);
        }
      }

      //
      // Wait for activity.
      //

      if (select(MaxFd + 1, &ReadSet_, &WriteSet_, nullptr, nullptr) == -1) {
        fmt::print("select failed with {}\n", SocketError());
        break;
      }

      //
      // Display some stats.
      //

      Stats_.Print();

      //
      // Scan the read set.
      //

      for (const auto &Fd : ReadFds) {

        //
        // If the Fd is not in the read set, let's continue.
        //

        if (!FD_ISSET(Fd, &ReadSet_)) {
          continue;
        }

        //
        // The server socket is a special case as it means that a client is
        // awaiting us to accept the connexion.
        //

        const bool IsServer = Fd == Server_;
        if (IsServer) {
          if (!HandleNewConnection()) {
            fmt::print("NewConnection failed\n");
            Ret = EXIT_FAILURE;
            break;
          }

          //
          // Once we handled the new connection we're done.
          //

          continue;
        }

        //
        // Otherwise, it means a client sent us a new result so handle that.
        //

        if (!HandleNewResult(Fd)) {

          //
          // If we failed handling of the result, let's just disconnect the
          // client.
          //

          if (!Disconnect(Fd)) {

            //
            // If we failed doing that... well, let's shut everything down.
            //

            fmt::print("Disconnect failed\n");
            Ret = EXIT_FAILURE;
            break;
          }
        }
      }

      //
      // Let's check if we have no more paths to handle and that we aren't going
      // above the number of mutations we are supposed to do.
      //

      if (Mutations_ >= Opts_.Runs && Paths_.size() == 0) {
        fmt::print("Completed {} mutations, time to stop the server..\n",
                   Mutations_);
        break;
      }

      //
      // Scan the write set.
      //

      for (const auto &Fd : WriteFds) {

        //
        // If the fd is not in the write set, let's continue.
        //

        if (!FD_ISSET(Fd, &WriteSet_)) {
          continue;
        }

        //
        // If the fd is in the write set, it means that it is ready to receive a
        // new testcase, so let's do that.
        // If we failed sending a new testcase over, then let's try to
        // disconnect the client.
        //

        if (!HandleNewRequest(Fd) && !Disconnect(Fd)) {

          //
          // If we failed to disconnect the client... let's just call it quits.
          //

          fmt::print("Disconnect failed\n");
          Ret = EXIT_FAILURE;
          break;
        }
      }
    }

    //
    // We exited, so let's force displaying the stats one last time.
    //

    Stats_.Print(true);
    return Ret;
  }

Run()의 전체 코드는 위와 같은데, 간단하게 요약하면 다음과 같이 나타낼 수 있다.

Run(){

    // Mutator 생성
    Mutator_ = std::make_unique<LibfuzzerMutator_t>(Rng_);

    ...

    // Input Directory 내의 모든 Corpus를 Path 에 등록
    fmt::print("Iterating through the corpus..\n");
    const fs::directory_iterator DirIt(Opts_.InputsPath);
    for (const auto &DirEntry : DirIt) {
      Paths_.emplace_back(DirEntry);
    }

    // File Size가 큰 Corpus 부터 시작되도록 정렬
    fmt::print("Sorting through the {} entries..\n", Paths_.size());
    std::sort(Paths_.rbegin(), Paths_.rend(), CompareTwoFileBySize);

    // Listen
    fmt::print("Running server on {}..\n", Opts_.Address);
    auto ServerOpt = Listen(Opts_.Address);
    if (!ServerOpt) {
      fmt::print("Listen() failed\n");
      return EXIT_FAILURE;
    }

    ...

    while (Ret == EXIT_SUCCESS) {


        // 모든 Client (Fuzz 노드)에 대한 Read/Write FD 등록
        for (const auto &[Fd, State] : Clients_) {
        MaxFd = std::max(MaxFd, Fd);
        if (State == SocketState_t::Read) {
          FD_SET(Fd, &ReadSet_);
          ReadFds.emplace_back(Fd);
        } else {
          FD_SET(Fd, &WriteSet_);
          WriteFds.emplace_back(Fd);
        }

        // 하나 선택
        if (select(MaxFd + 1, &ReadSet_, &WriteSet_, nullptr, nullptr) == -1) {
            fmt::print("select failed with {}\n", SocketError());
            break;
          }

        // Read 수행
          for (const auto &Fd : ReadFds) {
            // 서버의 경우 Accept 수행
            if (IsServer) {
                if (!HandleNewConnection()) {   //  <- Accept 수행
                fmt::print("NewConnection failed\n");
                Ret = EXIT_FAILURE;
                break;
              }
            ...
            // Client가 전송한 결과에 대하여 HandleNewResult() 함수에서 파싱함
            if (!HandleNewResult(Fd)) {   //  <- Coverage 및 Crash 결과 저장
                if (!Disconnect(Fd)) {
                    fmt::print("Disconnect failed\n");
                    Ret = EXIT_FAILURE;
                       break;
                }
            }
        }

        // Write 수행
        for (const auto &Fd : WriteFds) {
            // HandleNewRequest() 함수에서 Mutation한 새로운 Testcase를 생성함
            if (!HandleNewRequest(Fd) && !Disconnect(Fd)) {
              fmt::print("Disconnect failed\n");
              Ret = EXIT_FAILURE;
              break;
            }
        }

        ...
    }   
    ...
}

수행하는 동작만 요약하면 다음과 같음

• Input Directory 내의 모든 Corpus 찾기
• Server Listen
• 정상 종료까지 while()
  • Read 동작 - HandleNewResult()
    • Client가 전송한 Result(전송했던 Testcase, Coverage, Crash 등의 정보가 serialize된 데이터임)를 파싱함
    • 새로운 Coverage가 발견되면 전송했던 Testcase를 Corpus에 추가함
    • Crash 데이터가 전송되면 해당 데이터를 파일로 저장함
  • Write 동작 - HandleNewRequest()
    • 내부적으로 다음 두 가지의 함수를 호출함
    • GetTestcase() : 처음 호출되었을 때는 Input 디렉토리의 파일 데이터를 Testcase로 생성하고, Input 디렉토리 파일을 모두 Testcase로 사용한 이후에는 Corpus 내의 Testcase 중 하나를 선택하여 Mutate 후 Testcase를 생성함 Mutator의 경우 Libfuzzer의 Mutator를 그대로 사용함
    • SendTestcase() : Client로 Testcase를 전송함

LibFuzzer의 Mutator는 다음과 같이 다양한 MutationDispatcher 구현체가 있음

LibFuzzer Mutators

...
DefaultMutators.insert(
DefaultMutators.begin(),
{
   {&MutationDispatcher::Mutate_EraseBytes, "EraseBytes"},
   {&MutationDispatcher::Mutate_InsertByte, "InsertByte"},
   {&MutationDispatcher::Mutate_InsertRepeatedBytes,
    "InsertRepeatedBytes"},
   {&MutationDispatcher::Mutate_ChangeByte, "ChangeByte"},
   {&MutationDispatcher::Mutate_ChangeBit, "ChangeBit"},
   {&MutationDispatcher::Mutate_ShuffleBytes, "ShuffleBytes"},
   {&MutationDispatcher::Mutate_ChangeASCIIInteger, "ChangeASCIIInt"},
   {&MutationDispatcher::Mutate_ChangeBinaryInteger, "ChangeBinInt"},
   {&MutationDispatcher::Mutate_CopyPart, "CopyPart"},
   {&MutationDispatcher::Mutate_CrossOver, "CrossOver"},
   {&MutationDispatcher::Mutate_AddWordFromManualDictionary,
    "ManualDict"},
   {&MutationDispatcher::Mutate_AddWordFromPersistentAutoDictionary,
    "PersAutoDict"},
});
...
if (EF->LLVMFuzzerCustomMutator)
Mutators.push_back({&MutationDispatcher::Mutate_Custom, "Custom"});
else
Mutators = DefaultMutators;

if (EF->LLVMFuzzerCustomCrossOver)
Mutators.push_back(
 {&MutationDispatcher::Mutate_CustomCrossOver, "CustomCrossOver"});

HandleNewConnection()

  bool HandleNewConnection() {
    const SocketFd_t Client = accept(Server_, nullptr, nullptr);
    if (Client == INVALID_SOCKET) {
      fmt::print("accept failed\n");
      return false;
    }

    //
    // Keep track of the client.
    //

    Clients_.emplace(Client, SocketState_t::Write);
    Stats_.NewClient();
    Stats_.Print(true);
    return true;
  }

HandleNewResult()

  bool HandleNewResult(const SocketFd_t Fd) {

    //
    // Receive client data into the scratch buffer.
    //

    const auto ReceivedSize = Receive(Fd, ScratchBuffer_);
    if (!ReceivedSize) {
      fmt::print("Receive failed\n");
      return false;
    }

    //
    // Let's deserialize the response from the client.
    //

    const std::span<uint8_t> ReceivedBuffer(ScratchBuffer_.data(),
                                            *ReceivedSize);
    std::string ReceivedTestcase;
    tsl::robin_set<Gva_t> Coverage;
    TestcaseResult_t Result;
    if (!DeserializeResult(ReceivedBuffer, ReceivedTestcase, Coverage,
                           Result)) {
      fmt::print("DeserializeResult failed\n");
      return false;
    }

    //
    // If the client says has new coverage, then let's have a look.
    //

    if (Coverage.size() > 0) {

      //
      // Emplace the new coverage in our data.
      //
      const size_t SizeBefore = Coverage_.size();
      for (const auto &Gva : Coverage) {
        Coverage_.emplace(Gva);
      }
      //
      // If the coverage size has changed, it means that this testcase
      // provided new coverage indeed.
      //
      const bool NewCoverage = Coverage_.size() > SizeBefore;
      if (NewCoverage) {
        //
        // Allocate a test that will get moved into the corpus and maybe
        // saved on disk.
        //
        Testcase_t Testcase((uint8_t *)ReceivedTestcase.data(),
                            ReceivedTestcase.size());
        //
        // Before moving the buffer into the corpus, set up cross over with
        // it.
        //
        Mutator_->SetCrossOverWith(Testcase);
        //
        // Ready to move the buffer into the corpus now.
        //
        Corpus_.SaveTestcase(Result, std::move(Testcase));
      }
    }

    //
    // If the client reported a crash, let's check if it has a name, if so
    // we'll save it in the crashes folder.
    //

    if (const auto &Crash = std::get_if<Crash_t>(&Result)) {
      if (Crash->CrashName.size() > 0) {
        const auto &OutputPath = Opts_.CrashesPath / Crash->CrashName;
        const auto &Success =
            SaveFile(OutputPath, (uint8_t *)ReceivedTestcase.data(),
                     ReceivedTestcase.size());
        if (!Success) {
          fmt::print("Could not create the destination file.\n");
          return false;
        }

        const bool WroteFile = *Success;
        if (WroteFile) {
          fmt::print("Saving crash in {}\n", OutputPath.string());
        }
      }
    }

    //
    // We'll be waiting to be able to send this client a new job.
    //

    Clients_[Fd] = SocketState_t::Write;
    Stats_.Testcase(Result, Coverage_.size(), Corpus_.Size(), Corpus_.Bytes());
    return true;
  }

HandleNewRequest()

  
  bool HandleNewRequest(const SocketFd_t Fd) {

    //
    // Prepare a message to send to a client.
    //

    const std::string Testcase = GetTestcase();

    //
    // Send the testcase.
    //

    if (!SendTestcase(Fd, Testcase)) {
      fmt::print("SendTestcase failed\n");
      return false;
    }

    //
    // We'll be waiting for an answer from this client.
    //

    Clients_[Fd] = SocketState_t::Read;
    return true;
  }

  //
  // Deserializes a result from a client.
  //

  bool DeserializeResult(const std::span Buffer,
                         std::string &ReceivedTestcase,
                         tsl::robin_set &Coverage,
                         TestcaseResult_t &Result) {
    yas::mem_istream Is(Buffer.data(), Buffer.size_bytes());
    yas::binary_iarchive Ia(Is);
    Ia &ReceivedTestcase &Coverage &Result;
    return true;
  }

GetTestcase()

  std::string GetTestcase() {
    std::string TestcaseContent;

    //
    // If we have paths, it means we haven't finished to run through the corpus
    // yet, so this takes priority mutation stage.
    //

    if (Paths_.size() > 0) {

      //
      // Let's try to read a file.
      //

      std::unique_ptr<uint8_t[]> Buffer;
      size_t BufferSize = 0;

      //
      // We'll loop over the paths until we are able to read a file.
      //

      bool FoundFile = false;
      while (!FoundFile && Paths_.size() > 0) {

        //
        // Let's grab the smallest available file.
        //

        const auto &Path = Paths_.back();

        //
        // Let's try to read it.
        //

        Buffer = ReadFile(Path, BufferSize);

        //
        // If reading the file failed for whatever reasons, or if the file is
        // too big, we're looping and trying again.
        //

        const bool Valid =
            BufferSize > 0 && BufferSize <= Opts_.TestcaseBufferMaxSize;

        if (!Valid) {
          fmt::print("Skipping because {} size is zero or bigger than the max "
                     "({} vs {})\n",
                     Path.string(), BufferSize, Opts_.TestcaseBufferMaxSize);
        }

        //
        // We are done with this path now!
        //

        Paths_.pop_back();

        if (!Valid) {
          continue;
        }

        //
        // Phew, we read a file \o/
        //

        FoundFile = true;
      }

      //
      // If we have read a file successfully in the previous stage, this is the
      // testcase we'll return to the caller.
      //

      if (FoundFile) {
        TestcaseContent.resize(BufferSize);
        memcpy(TestcaseContent.data(), Buffer.get(), BufferSize);
        return TestcaseContent;
      }
    }

    //
    // If we get here, it means that we are ready to mutate.
    // First thing we do is to grab a seed.
    //

    const Testcase_t *Testcase = Corpus_.PickTestcase();
    if (!Testcase) {
      fmt::print("The corpus is empty, exiting\n");
      std::abort();
    }

    //
    // If the testcase is too big, abort as this should not happen.
    //

    if (Testcase->BufferSize_ > Opts_.TestcaseBufferMaxSize) {
      fmt::print(
          "The testcase buffer len is bigger than the testcase buffer max "
          "size.\n");
      std::abort();
    }

    //
    // Copy the input in a buffer we're going to mutate.
    //

    memcpy(ScratchBuffer_.data(), Testcase->Buffer_.get(),
           Testcase->BufferSize_);

    //
    // Mutate in the scratch buffer.
    //

    const size_t TestcaseBufferSize =
        Mutator_->Mutate(ScratchBuffer_.data(), Testcase->BufferSize_,
                         Opts_.TestcaseBufferMaxSize);

    //
    // Copy the testcase in its own buffer before sending it to the
    // consumer.
    //

    TestcaseContent.resize(TestcaseBufferSize);
    memcpy(TestcaseContent.data(), ScratchBuffer_.data(), TestcaseBufferSize);
    Mutations_++;
    return TestcaseContent;
  }

ScratchBuffer는 앞서 Run()에서 선언 및 초기화했던 버퍼이며, Mutation할 때 결과를 담기위한 임시 버퍼임

Testcase에 대한 데이터를 저장하는 버퍼는 TestcaseContent 객체임

SendTestcase()

  
  bool SendTestcase(const SocketFd_t &Fd, const std::string &Testcase) {

    //
    // Serialize the message.
    //

    yas::mem_ostream Os;
    yas::binary_oarchive Oa(Os);
    Oa &Testcase;
    const auto &Buf = Os.get_intrusive_buffer();
    if (!Send(Fd, (uint8_t *)Buf.data, Buf.size)) {
      fmt::print("Send failed\n");
      return false;
    }

    return true;
  }

Fuzz (=Client) Node

CLI 라이브러리를 통해 인자를 알맞게 파싱한 다음, 몇 가지 초기화 동작을 수행하는데, backend에 대한 부분이 핵심인 것 같다.

단순하게 보면 Memroy dump 데이터와 CPU State 정보만 있으면 vm을 실행시킬 수 있는 것 같다. Json 형태의 CPU State 데이터를 파싱하여 CpuState 객체에 저장하고, 해당 데이터와 Memory Dump 파일 데이터를 바탕으로 각 backend (Windows HyperVisor, bockcpu, KVM)의 Initialize 및 Restore 작업을 진행한다.

  CpuState_t CpuState;
  if (!LoadCpuStateFromJSON(CpuState, Opts.CpuStatePath)) {
    fmt::print("LoadCpuStateFromJSON failed, no take off today.\n");
    return EXIT_FAILURE;
  }

#ifdef WINDOWS
  if (Opts.Backend == BackendType_t::Whv) {
    g_Backend = new WhvBackend_t();
  }
#endif
#ifdef LINUX
  if (Opts.Backend == BackendType_t::Kvm) {
    g_Backend = new KvmBackend_t();
  }
#endif
  if (Opts.Backend == BackendType_t::Bochscpu) {
    g_Backend = new BochscpuBackend_t();
  }

  //
  // Initialize the debugger instance.
  //

  if (!g_Dbg.Init(Opts.DumpPath, Opts.SymbolFilePath)) {
    return EXIT_FAILURE;
  }

  //
  // Set an instruction limit to avoid infinite loops, etc.
  //

  if (Opts.Limit != 0) {
    g_Backend->SetLimit(Opts.Limit);
  }

  //
  // Initialize the backend with a state. This ensures the backend is ready to
  // service memory / register access, etc.
  //
  // Because SanitizeCpuState needs to read virtual memory, the backend has to
  // start from somewhere. We first flush the state as is and this should be
  // enough to have SanitizeCpuState do its job.
  //

  if (!g_Backend->Initialize(Opts, CpuState)) {
    fmt::print("Backend failed initialization.\n");
    return EXIT_FAILURE;
  }

  //
  // Sanitize the state before running.
  //

  if (!SanitizeCpuState(CpuState)) {
    fmt::print("SanitizeCpuState failed, no take off today.\n");
    return EXIT_FAILURE;
  }

  //
  // We now have the real starting state we want to start with, so we make sure
  // it gets set in the backend and to do that we call the Restore function.
  // This ensures we start from a clean state.
  //

  if (!g_Backend->Restore(CpuState)) {
    fmt::print("Backend failed to restore.\n");
    return EXIT_FAILURE;
  }

  //
  // Now invoke the fuzz command if this is what we want.
  //

  if (Wtf.got_subcommand("fuzz")) {
    return FuzzSubcommand(Opts, *Target, CpuState);
  }

FuzzSubcommand()

int FuzzSubcommand(const Options_t &Opts, const Target_t &Target,
                   const CpuState_t &CpuState) {
  return Client_t(Opts).Run(Target, CpuState);
}

Client_t

class Client_t {
  SocketFd_t Client_ = INVALID_SOCKET;
  const Options_t &Opts_;
  chrono::high_resolution_clock::time_point Start_ =
      chrono::high_resolution_clock::now();
  chrono::high_resolution_clock::time_point LastPrint_ =
      chrono::high_resolution_clock::now();
  uint64_t Received_ = 0;
  std::unique_ptr<uint8_t[]> Scratch_;
  std::span<uint8_t> ScratchBuffer_;
    ...

Client_t->Run()

여기서 Target은 Harness를 가리킴

int Client_t::Run(const Target_t &Target, const CpuState_t &CpuState) {

  // harness Init 호출함
  if (!Target.Init(Opts_, CpuState)) {
    fmt::print("Failed to initialize the target\n");
    return EXIT_FAILURE;
  }

  // Server에 연결 tcp://localhost:31337/
  fmt::print("Dialing to {}..\n", Opts_.Fuzz.Address);
  auto ClientOpt = Dial(Opts_.Fuzz.Address);
  if (!ClientOpt) {
    fmt::print("Dial failed\n");
    return EXIT_FAILURE;
  }

  Client_ = *ClientOpt;

  while (1) {
    const auto ReceivedSize =
        Receive(Client_, ScratchBuffer_.data(), ScratchBuffer_.size_bytes());
    if (!ReceivedSize) {
      fmt::print("Receive failed\n");
      break;
    }

    //
    // Deserialize the testcase.
    //

    const std::string Testcase =
        DeserializeTestcase({ScratchBuffer_.data(), *ReceivedSize});

    //
    // Run the testcase.
    //

    const TestcaseResult_t TestcaseResult = RunTestcaseAndRestore(
        Target, CpuState, {(uint8_t *)Testcase.data(), Testcase.size()});

    //
    // If we triggered a time out testcase, we ask the backend to invalidate
    // the new coverage it has triggered. On top of that, we lie to the
    // server by telling it that we haven't hit new coverage.
    //

    if (std::holds_alternative<Timedout_t>(TestcaseResult)) {
      g_Backend->RevokeLastNewCoverage();
    }

    //
    // Send the result back to the server.
    //

    if (!SendResult(Client_, Testcase, g_Backend->LastNewCoverage(),
                    TestcaseResult)) {
      fmt::print("SendResult failed\n");
      break;
    }

    Received_++;
  }

  g_Stats.Print(true);
  CloseSocket(Client_);
  return EXIT_SUCCESS;
}

• Target (harness)의 Init() 함수를 호출함
• Dial을 이용하여 Server와 연결 (ex. tcp://localhost:31337/)
• while 반복
  • 서버로부터 전송된 데이터를 Deserialize하여 Testcase 데이터를 생성함
  • RunTestcaseAndRestore()를 호출하여 해당 Testcase로 VM 실행
  • 해당 Testcase가 타임아웃된 경우 g_Backend->RevokeLastNewCoverage()를 호출하여 Coverage를 제거함
  • Server에게 Testcase, Coverage 등 결과를 전송함

Target.Init()

bool Init(const Options_t &Opts, const CpuState_t &) {
  //
  // Stop the test-case once we return back from the call [DeviceIoControl]
  //

  const Gva_t Rip = Gva_t(g_Backend->Rip());
  const Gva_t AfterCall = Rip + Gva_t(6);
  if (!g_Backend->SetBreakpoint(AfterCall, [](Backend_t *Backend) {
        DebugPrint("Back from kernel!\n");
        Backend->Stop(Ok_t());
      })) {
    DebugPrint("Failed to SetBreakpoint AfterCall\n");
    return false;
  }

  //
  // NOP the calls to DbgPrintEx.
  //

  if (!g_Backend->SetBreakpoint("nt!DbgPrintEx", [](Backend_t *Backend) {
        const Gva_t FormatPtr = Backend->GetArgGva(2);
        const std::string &Format = Backend->VirtReadString(FormatPtr);
        DebugPrint("DbgPrintEx: {}", Format);
        Backend->SimulateReturnFromFunction(0);
      })) {
    DebugPrint("Failed to SetBreakpoint DbgPrintEx\n");
    return false;
  }

  //
  // Make ExGenRandom deterministic.
  //
  // kd> ub fffff805`3b8287c4 l1
  // nt!ExGenRandom+0xe0:
  // fffff805`3b8287c0 480fc7f2        rdrand  rdx
  const Gva_t ExGenRandom = Gva_t(g_Dbg.GetSymbol("nt!ExGenRandom") + 0xe4);
  if (!g_Backend->SetBreakpoint(ExGenRandom, [](Backend_t *Backend) {
        DebugPrint("Hit ExGenRandom!\n");
        Backend->Rdx(Backend->Rdrand());
      })) {
    return false;
  }

  //
  // Catch bugchecks.
  //

  if (!g_Backend->SetBreakpoint("nt!KeBugCheck2", [](Backend_t *Backend) {
        const uint64_t B0 = g_Backend->GetArg(1);
        const uint64_t B1 = g_Backend->GetArg(2);
        const uint64_t B2 = g_Backend->GetArg(3);
        const uint64_t B3 = g_Backend->GetArg(4);
        const uint64_t B4 = g_Backend->GetArg(5);
        const std::string Filename = fmt::format(
            "crash-{:#x}-{:#x}-{:#x}-{:#x}-{:#x}", B0, B1, B2, B3, B4);
        DebugPrint("KeBugCheck2: {}\n", Filename);
        Backend->Stop(Crash_t(Filename));
      })) {
    return false;
  }

  //
  // Catch context-switches.
  //

  if (!g_Backend->SetBreakpoint("nt!SwapContext", [](Backend_t *Backend) {
        DebugPrint("nt!SwapContext\n");
        Backend->Stop(Cr3Change_t());
      })) {
    return false;
  }

  return true;
}

Target.InsertTestcase()

bool InsertTestcase(const uint8_t *Buffer, const size_t BufferSize) {
  if (BufferSize < sizeof(uint32_t)) {
    return true;
  }

  const uint32_t Ioctl = *(uint32_t *)Buffer;
  const size_t IoctlBufferSize = BufferSize - sizeof(uint32_t);
  const uint8_t *IoctlBuffer = Buffer + sizeof(uint32_t);
  if (IoctlBufferSize > 1024) {
    return true;
  }

  //  DeviceIoControl(
  //    H,
  //    0xdeadbeef,
  //    Buffer.data(),
  //    Buffer.size(),
  //    Buffer.data(),
  //    Buffer.size(),
  //    &Returned,
  //    nullptr
  //);

  g_Backend->Rdx(Ioctl);
  const Gva_t IoctlBufferPtr = Gva_t(g_Backend->R8());
  if (!g_Backend->VirtWriteDirty(IoctlBufferPtr, IoctlBuffer,
                                 IoctlBufferSize)) {
    DebugPrint("VirtWriteDirty failed\n");
    return false;
  }

  g_Backend->R9(IoctlBufferSize);
  const Gva_t Rsp = Gva_t(g_Backend->Rsp());
  const Gva_t OutBufferSizePtr = Rsp + Gva_t(4 * sizeof(uint64_t));
  if (!g_Backend->VirtWriteStructDirty(OutBufferSizePtr, &IoctlBufferSize)) {
    DebugPrint("VirtWriteStructDirty failed\n");
    return false;
  }

  return true;
}

RunTestcaseAndRestore()

TestcaseResult_t RunTestcaseAndRestore(const Target_t &Target,
                                       const CpuState_t &CpuState,
                                       const std::span<uint8_t> Buffer,
                                       const bool PrintRunStats) {
  //
  // Let the stats know that we are about to start to execute a testcase.
  //

  g_Stats.TestcaseStarts();

  //
  // Invoke the user callback so that it can insert the testcase.
  //

  if (!Target.InsertTestcase(Buffer.data(), Buffer.size_bytes())) {
    fmt::print("Failed to insert testcase\n");
    std::abort();
  }

  //
  // Run the testcase.
  //

  const auto &Res = g_Backend->Run(Buffer.data(), Buffer.size_bytes());
  if (!Res) {
    fmt::print("Failed to run the testcase\n");
    std::abort();
  }

  //
  // Let know the stats that we finished a testcase.
  //

  const auto &LastNewCoverage = g_Backend->LastNewCoverage();
  g_Stats.TestcaseEnds(*Res, LastNewCoverage.size());

  //
  // Let the stats that we are about to start a restore.
  //

  g_Stats.RestoreStarts();

  //
  // Invoke the user callback to give it a chance to restore things.
  //

  if (!Target.Restore()) {
    fmt::print("Failed to restore\n");
    std::abort();
  }

  //
  // Restore the execution environment.
  //

  if (!g_Backend->Restore(CpuState)) {
    fmt::print("Failed to restore the backend\n");
    std::abort();
  }

  //
  // Let the stats that we finished restoring.
  //

  g_Stats.RestoreEnds();

  //
  // Print the run stats after restoring because some backend only know about
  // dirty pages at restore time.
  //

  if (PrintRunStats) {
    g_Backend->PrintRunStats();
  }

  //
  // Print the global stats.
  //

  g_Stats.Print(PrintRunStats);
  return *Res;
}

g_Backend->Run()

std::optional<TestcaseResult_t>
BochscpuBackend_t::Run(const uint8_t *Buffer, const uint64_t BufferSize) {

  //
  // Initialize a few things.
  //

  TestcaseBuffer_ = Buffer;
  TestcaseBufferSize_ = BufferSize;
  LastNewCoverage_.clear();

  //
  // Reset some of the stats.
  //

  RunStats_.Reset();

  //
  // Lift off.
  //

  bochscpu_cpu_run(Cpu_, HookChain_);

  //
  // Fill in the stats.
  //

  RunStats_.AggregatedCodeCoverage = AggregatedCodeCoverage_.size();
  RunStats_.DirtyGpas = DirtyGpas_.size();

  //
  // Return to the user how the testcase exited.
  //

  return TestcaseResult_;
}

ㄴ백엔드 부분도 이어서...