From 7ffbf4fab8dc4ce97d740692a1ae232d3ccbb104 Mon Sep 17 00:00:00 2001 From: Jaeseung Choi Date: Sat, 19 Sep 2020 20:14:24 +0900 Subject: [PATCH 01/29] Simplify to support stdin/file input fuzzing only --- src/Core/ByteVal.fs | 5 +- src/Core/Config.fs | 4 +- src/Core/Input.fs | 253 -------------- src/Core/Seed.fs | 477 +++++++++----------------- src/Core/TestCase.fs | 42 --- src/Core/Typedef.fs | 47 +-- src/Core/Utils.fs | 9 +- src/Eclipser.fsproj | 5 - src/Executor/Executor.fs | 240 ++++++------- src/Fuzz/Fuzz.fs | 45 +-- src/Fuzz/GreyConcolic/GreyConcolic.fs | 2 +- src/Fuzz/GreyConcolic/Solve.fs | 12 +- src/Fuzz/Initialize.fs | 75 +--- src/Fuzz/Manager.fs | 32 +- src/Fuzz/Options.fs | 45 +-- src/Fuzz/RandomFuzz.fs | 19 +- src/Fuzz/SeedQueue.fs | 3 +- src/Main/Decode.fs | 67 ---- src/Main/Main.fs | 25 -- src/Main/Replay.fs | 62 ---- 20 files changed, 351 insertions(+), 1118 deletions(-) delete mode 100644 src/Core/TestCase.fs delete mode 100644 src/Main/Decode.fs delete mode 100644 src/Main/Main.fs delete mode 100644 src/Main/Replay.fs diff --git a/src/Core/ByteVal.fs b/src/Core/ByteVal.fs index 81a5ce8..fcd6b44 100644 --- a/src/Core/ByteVal.fs +++ b/src/Core/ByteVal.fs @@ -53,11 +53,10 @@ module ByteVal = sprintf "%02x@(%02x-%02x)" ((low + upper) / 2uy) low upper | Sampled b -> sprintf "%02x*" b - let getMinMax byteVal inputKind = + let getMinMax byteVal inputSrc = match byteVal with | Untouched _ | Undecided _ | Sampled _ -> - if inputKind = InputKind.Args || inputKind = InputKind.StdIn - then (0uy, 127uy) (* Slightly over-approximated printable bytes *) + if inputSrc = StdInput then (0uy, 127uy) (* Printable bytes *) else (0uy, 255uy) (* All bytes *) | Fixed x -> (x, x) | Interval (low, upper) -> (low, upper) diff --git a/src/Core/Config.fs b/src/Core/Config.fs index dc74649..5eb8a98 100644 --- a/src/Core/Config.fs +++ b/src/Core/Config.fs @@ -30,7 +30,9 @@ let CtxSensitivity = 0 /// The length of each input during the initialization of a seed. If the user /// explicitly provided initial seed inputs, this parameter will not be used. -let InitInputLen = 16 +let INIT_INPUT_LEN = 16 + +let MAX_INPUT_LEN = 1048576 let DurableQueueMaxSize = 1000 diff --git a/src/Core/Input.fs b/src/Core/Input.fs index 99298b1..31699cf 100644 --- a/src/Core/Input.fs +++ b/src/Core/Input.fs @@ -5,256 +5,3 @@ open Config open Utils open BytesUtils -/// An input that consists of an array of ByteVals. -type Input = { - /// An array of ByteVal elements. - ByteVals : ByteVal array - /// Maximum length allowed for this input. - MaxLen : int - /// Specifies the offset within an input (i.e. the index of 'ByteVals'), which - /// will be used for the next grey-box concolic testing. - ByteCursor : int - /// The direction in which ByteCursor should move. - CursorDir : Direction -} - -module Input = - - /// An empty input. - let empty = { ByteVals = [| |]; MaxLen = 0; ByteCursor = 0; CursorDir = Stay } - - /// Initialize an input for the specified input source, with the specified - /// maximum length. - let make inputSrc maxLen = - let initLen = min maxLen InitInputLen - let initByte = match inputSrc with - | Args | StdIn -> 65uy // Character 'A'. - | File -> 0uy // NULL byte. - let bytes = Array.init initLen (fun _ -> initByte) - let byteVals = Array.map ByteVal.newByteVal bytes - { ByteVals = byteVals; MaxLen = maxLen; ByteCursor = 0; CursorDir = Right } - - /// Initialize an input with provided byte array content. - let makeWith bytes maxLen = - // Do not allow empty content. - if Array.length bytes = 0 then failwith "Input.makeWith() with empty bytes" - let byteVals = Array.map ByteVal.newByteVal bytes - { ByteVals = byteVals; MaxLen = maxLen; ByteCursor = 0; CursorDir = Right } - - /// Concretize an input into a byte array. - let concretize input = - Array.map ByteVal.getConcreteByte input.ByteVals - - /// Concretize an input into a byte array, until the first NULL byte is met. - let concretizeArg input = - let bytes = concretize input - let idxOpt = Array.tryFindIndex (fun b -> b = 0uy) bytes - match idxOpt with - | None -> bytesToStr bytes - | Some idx -> bytesToStr bytes.[0 .. (idx - 1)] - - /// Get current byte value pointed by input's cursor. - let getCurByteVal input = input.ByteVals.[input.ByteCursor] - - /// Check if the given input has any unfixed ByteVal. - let hasUnfixedByte input = Array.exists ByteVal.isUnfixed input.ByteVals - - /// Return the index of the first unfixed ByteVal. Raises an exception if - /// unfixed ByteVal do not exists, so hasUnfixedByte() should precede. - let getUnfixedByteIndex input = - Array.findIndex ByteVal.isUnfixed input.ByteVals - - /// Set the byte cursor position of the given input. - let setCursor input newPos = { input with ByteCursor = newPos } - - /// Set the byte cursor direction of the given input - let setCursorDir input dir = { input with Input.CursorDir = dir } - - /// Step the byte cursor, following the cursor direction. - let stepCursor input = - let byteCursor = input.ByteCursor - match input.CursorDir with - | Stay -> None - | Left when 0 <= byteCursor - 1 -> - Some (setCursor input (byteCursor - 1)) - | Right when byteCursor + 1 < input.ByteVals.Length -> - Some (setCursor input (byteCursor + 1)) - | Left _ | Right _ -> None - - // Starting from 'curIdx', find the index of the first unfixed ByteVal. - let rec private findUnfixedByte (bytes: ByteVal []) curIdx = - if curIdx < 0 || curIdx >= bytes.Length then -1 - elif ByteVal.isUnfixed bytes.[curIdx] then curIdx - else findUnfixedByte bytes (curIdx + 1) - - // Starting from 'curIdx', find the index of the first unfixed ByteVal, in a - // backward direction. - let rec private findUnfixedByteBackward (bytes: ByteVal []) curIdx = - if curIdx < 0 || curIdx >= bytes.Length then -1 - elif ByteVal.isUnfixed bytes.[curIdx] then curIdx - else findUnfixedByteBackward bytes (curIdx - 1) - - /// Move the byte cursor to an unfixed ByteVal. Cursor may stay at the same - /// position. - let moveToUnfixedByte input = - let byteCursor = input.ByteCursor - let cursorDir = input.CursorDir - match cursorDir with - | Stay -> None - | Left -> let offset = findUnfixedByteBackward input.ByteVals byteCursor - if offset <> -1 then Some (setCursor input offset) else None - | Right -> let offset = findUnfixedByte input.ByteVals byteCursor - if offset <> -1 then Some (setCursor input offset) else None - - /// Move the byte cursor to the next unfixed ByteVal. Cursor should move at - /// least one offset toward the current cursor direction. - let proceedCursor input = - match stepCursor input with - | None -> None - | Some newInput -> moveToUnfixedByte newInput - - // Auxiliary function for byteValsToStr() that handles 'Untouched' ByteVals. - let private untouchedToStr untouchedList = - if List.isEmpty untouchedList then "" - elif List.length untouchedList < 4 then - " " + String.concat " " (List.map ByteVal.toString untouchedList) - else sprintf " ...%dbytes..." (List.length untouchedList) - - // Stringfy ByteVal list. - let rec private byteValsToStr accumUntouched accumStrs byteVals = - match byteVals with - | [] -> accumStrs + untouchedToStr (List.rev accumUntouched) - | headByteVal :: tailByteVals -> - (match headByteVal with - | Untouched _ -> // Just accumulate to 'accumUntouched' and continue - byteValsToStr (headByteVal :: accumUntouched) accumStrs tailByteVals - | Undecided _ | Fixed _ | Interval _ | Sampled _ -> - let untouchedStr = untouchedToStr (List.rev accumUntouched) - let byteValStr = ByteVal.toString headByteVal - let accumStrs = accumStrs + untouchedStr + " " + byteValStr - byteValsToStr [] accumStrs tailByteVals) // reset accumUntouched to [] - - /// Stringfy an input. - let toString input = - let byteVals = List.ofArray input.ByteVals - let byteStr = byteValsToStr [] "" byteVals - sprintf "%s (%d) (%A)" byteStr input.ByteCursor input.CursorDir - - -/// A sequence of inputs. For example, a command-line argument is represented -/// with an InputSeq, where each input corresponds to each string element of -/// argv[] array. -type InputSeq = { - /// An array of Inputs. - Inputs : Input array - /// Specifies the offset within a sequence (i.e. the index of 'Inputs'), which - /// will be used for the next grey-box concolic testing. - InputCursor : int - /// The direction in which CursorPos should move. - CursorDir : Direction -} - -module InputSeq = - /// An empty input sequence. - let empty = { Inputs = [| |]; InputCursor = 0; CursorDir = Stay } - - /// Initialize an input sequence for the specified input source, with the - /// specified maximum lengths. - let make inputSrc maxLens = - let inputs = Array.ofList maxLens - |> Array.filter (fun len -> len <> 0) - |> Array.map (Input.make inputSrc) - { Inputs = inputs; InputCursor = 0; CursorDir = Right } - - /// Concretize an input sequence into an array of 'byte array'. - let concretize inputSeq = - Array.map Input.concretize inputSeq.Inputs - - /// Concretize an argument input sequence into a string array. - let concretizeArg inputSeq = - Array.map Input.concretizeArg inputSeq.Inputs - |> Array.filter (fun str -> str <> "") // Filter out empty string args - - let isEmpty inputSeq = - inputSeq.Inputs.Length = 0 - - let isSingular inputSeq = - inputSeq.Inputs.Length = 1 - - /// Get the current input pointed by cursor. - let getCurInput inputSeq = inputSeq.Inputs.[inputSeq.InputCursor] - - /// Update (replace) the current input pointed by cursor. - let setCurInput inputSeq newInput = - let newInputs = Array.copy inputSeq.Inputs - newInputs.[inputSeq.InputCursor] <- newInput - { inputSeq with Inputs = newInputs } - - /// Set the input cursor position of the given input sequence. - let setCursor inputSeq newPos = - { inputSeq with InputCursor = newPos} - - /// Set the input cursor direction of the given input sequence. - let setCursorDir inputSeq dir = - { inputSeq with InputSeq.CursorDir = dir } - - /// Step the input cursor toward currently pointing cursor direction. - let stepCursor inputSeq = - let cursorPos = inputSeq.InputCursor - let cursorDir = inputSeq.CursorDir - match cursorDir with - | Left when 0 <= cursorPos - 1 -> - Some (setCursor inputSeq (cursorPos - 1)) - | Right when cursorPos + 1 < inputSeq.Inputs.Length -> - Some (setCursor inputSeq (cursorPos + 1)) - | Stay | Left _ | Right _ -> None - - // Find an input that has any unfixed byte, starting from the given index. - // If such input is found, return a new input with updated byte cursor, along - // with the index of the found input. - let rec private findUnfixedInput (inputs: Input []) curIdx = - if curIdx < 0 || curIdx >= inputs.Length then None else - let curInput = inputs.[curIdx] - if Input.hasUnfixedByte curInput then - let unfixedOffset = Input.getUnfixedByteIndex curInput - let newInput = Input.setCursor curInput unfixedOffset - |> Input.setCursorDir <| Right - Some (newInput, curIdx) - else findUnfixedInput inputs (curIdx + 1) - - // Find an input that has any unfixed byte, starting from the given index, in - // a backward direction. If such input is found, return a new input with - // updated byte cursor, along with the index of the found input. - let rec private findUnfixedInputBackward (inputs: Input []) curIdx = - if curIdx < 0 || curIdx >= inputs.Length then None else - let curInput = inputs.[curIdx] - if Input.hasUnfixedByte curInput then - let unfixedOffset = Input.getUnfixedByteIndex curInput - let newInput = Input.setCursor curInput unfixedOffset - |> Input.setCursorDir <| Right - Some (newInput, curIdx) - else findUnfixedInputBackward inputs (curIdx - 1) - - /// From a given input sequence, find an input that has any unfixed ByteVal, - /// and return a new input sequence with updated input/byte cursors. The - /// cursor may stay at the same position. - let moveToUnfixedInput inputSeq = - let inputCursor = inputSeq.InputCursor - let newInputOpt = - match inputSeq.CursorDir with - | Stay -> None - | Left -> findUnfixedInputBackward inputSeq.Inputs inputCursor - | Right -> findUnfixedInput inputSeq.Inputs inputCursor - match newInputOpt with - | None -> None - | Some (newInput, inputIdx) -> - let newInputSeq = setCurInput (setCursor inputSeq inputIdx) newInput - Some newInputSeq - - /// Proceed the byte cursor to the next input that has any unfixed ByteVal. - /// Cursor should move at least one offset toward the current cursor - /// direction. - let proceedCursor inputSeq = - match stepCursor inputSeq with - | None -> None - | Some newInputSeq -> moveToUnfixedInput newInputSeq diff --git a/src/Core/Seed.fs b/src/Core/Seed.fs index 5670faa..5eacbda 100644 --- a/src/Core/Seed.fs +++ b/src/Core/Seed.fs @@ -1,128 +1,93 @@ namespace Eclipser -open System open Config open Utils -open BytesUtils -type InputSrc = - | StdInput - | FileInput of int (* argv[] index *) - -/// Represents the file path, which can be either an index of command-line -/// argument, or a constant string. -type Filepath = ArgIdx of int | ConstPath of string | Unknown - -/// FileInput is an 'Input' coupled with its file path. -type FileInput = { - Path : Filepath - Content : InputSeq -} - -/// Seed represents an input to a program, along with various information (e.g. -/// approximate path conditions, cursor, etc.) needed for test case generation. +/// An input that consists of an array of ByteVals. type Seed = { - /// Command line arguments represented with a sequence of input. - Args : InputSeq - /// Standard input. Currently, we do not consider a sequence of inputs. - StdIn : InputSeq - /// File input. Currently, we do not consider multiple file inputs. - File : FileInput - /// Specifies input source that will be used for the next grey-box concolic - /// testing. - SourceCursor : InputKind + /// An array of ByteVal elements. + ByteVals : ByteVal array + /// Indes of the byte to be used for the next grey-box concolic testing. + CursorPos : int + /// The direction in which ByteCursor should move. + CursorDir : Direction + /// Input source. + Source : InputSource } module Seed = - /// Initialize a seed with specified maximum lengths. - let make inputSrc argMaxLens fileMaxLen stdInMaxLen = - { Args = InputSeq.make Args argMaxLens - StdIn = InputSeq.make StdIn [stdInMaxLen] - File = { Path = Unknown; Content = InputSeq.make File [fileMaxLen] } - SourceCursor = inputSrc } + let dummy = { + ByteVals = [| |] + CursorPos = 0 + CursorDir = Right + Source = StdInput + } - /// Initialize a seed with the specified content bytes, input source and + /// Initialize a seed for the specified input source, with the specified /// maximum length. - let makeWith inputSrc maxLen (initBytes: byte[]) = - let initInput = Input.makeWith initBytes maxLen - let initInputSeq = - { Inputs = [| initInput |]; InputCursor = 0; CursorDir = Right } - let argInput = if inputSrc = Args then initInputSeq else InputSeq.empty - let fileContent = if inputSrc = File then initInputSeq else InputSeq.empty - let fileInput = { Path = Unknown; Content = fileContent } - let stdIn = if inputSrc = StdIn then initInputSeq else InputSeq.empty - { Args = argInput - StdIn = stdIn - File = fileInput - SourceCursor = inputSrc } - - /// Concretize input file path into a string. - let concretizeFilepath seed = - let argStrs = InputSeq.concretizeArg seed.Args - match seed.File.Path with - | ArgIdx i -> argStrs.[i] - | ConstPath path -> path - | Unknown -> "" + let make src = + let initByte = match src with + | StdInput -> 65uy // Character 'A'. + | FileInput _ -> 0uy // NULL byte. + let bytes = Array.init INIT_INPUT_LEN (fun _ -> initByte) + let byteVals = Array.map ByteVal.newByteVal bytes + { ByteVals = byteVals; CursorPos = 0; CursorDir = Right; Source = src } + + /// Initialize a seed with provided byte array content. + let makeWith src bytes = + // Do not allow empty content. + if Array.length bytes = 0 then failwith "Seed.makeWith() with empty bytes" + let byteVals = Array.map ByteVal.newByteVal bytes + { ByteVals = byteVals; CursorPos = 0; CursorDir = Right; Source = src } + + /// Concretize a seed into a byte array. + let concretize seed = + Array.map ByteVal.getConcreteByte seed.ByteVals (**************************** Getter functions ****************************) - /// Get the current input sequence pointed by the cursor. - let getCurInputSeq seed = - match seed.SourceCursor with - | Args -> seed.Args - | StdIn -> seed.StdIn - | File -> seed.File.Content - - /// Get the current input pointed by the cursor. - let getCurInput seed = - let curInputSeq = getCurInputSeq seed - InputSeq.getCurInput curInputSeq + /// Get the current ByteVal pointed by the cursor. + let getCurByteVal seed = seed.ByteVals.[seed.CursorPos] - let getByteCursorDir seed = - let curInput = getCurInput seed - curInput.CursorDir + /// Get the length of byte values. + let getCurLength seed = seed.ByteVals.Length - /// Get the length of current input pointed by the cursor. - let getCurInputLen seed = - let curInput = getCurInput seed - curInput.ByteVals.Length + /// Return the index of the first unfixed ByteVal. Raises an exception if + /// unfixed ByteVal do not exists, so hasUnfixedByte() should precede. + let getUnfixedByteIndex seed = + Array.findIndex ByteVal.isUnfixed seed.ByteVals - /// Get the current ByteVal pointed by the cursor. - let getCurByteVal seed = - let curInput = getCurInput seed - Input.getCurByteVal curInput + /// Get the direction of the cursor. + let getByteCursorDir seed = + seed.CursorDir /// Get the concrete value of the ByteVal at the specified offset of the - /// current input. + /// current seed. let getConcreteByteAt seed pos = - let curInput = getCurInput seed - let curByteVals = curInput.ByteVals - ByteVal.getConcreteByte curByteVals.[pos] + ByteVal.getConcreteByte seed.ByteVals.[pos] /// Get the concrete values of ByteVals starting from the specified offset of - /// the current input. + /// the current seed. let getConcreteBytesFrom seed pos len = - let curInput = getCurInput seed - let curByteVals = curInput.ByteVals - Array.map ByteVal.getConcreteByte curByteVals.[pos .. (pos + len - 1)] + Array.map ByteVal.getConcreteByte seed.ByteVals.[pos .. (pos + len - 1)] (**************************** Query functions ****************************) - /// Check if the byte at the given offset of current input is unfixed. + /// Check if the given seed has any unfixed ByteVal. + let hasUnfixedByte seed = Array.exists ByteVal.isUnfixed seed.ByteVals + + /// Check if the byte at the given offset of current seed is unfixed. let isUnfixedByteAt seed offset = - let curInput = getCurInput seed - ByteVal.isUnfixed curInput.ByteVals.[offset] + ByteVal.isUnfixed seed.ByteVals.[offset] /// Find the remaining length toward the given direction, starting from the /// current byte position. let queryLenToward seed direction = - let curInput = getCurInput seed - let bytePos = curInput.ByteCursor match direction with | Stay -> failwith "queryLenToward() cannot be called with 'Stay'" - | Right -> curInput.ByteVals.Length - bytePos - | Left -> bytePos + 1 + | Right -> seed.ByteVals.Length - seed.CursorPos + | Left -> seed.CursorPos + 1 // Auxiliary function for queryUpdateBound() let private queryUpdateBoundLeft (byteVals: ByteVal []) byteCursor = @@ -148,19 +113,17 @@ module Seed = /// Find the maximum length that can be updated for grey-box concolic testing. let queryUpdateBound seed direction = - let curInput = getCurInput seed - let byteVals = curInput.ByteVals - let byteCursor = curInput.ByteCursor + let byteVals = seed.ByteVals + let byteCursor = seed.CursorPos match direction with | Stay -> failwith "queryUpdateBound() cannot be called with 'Stay'" | Left -> queryUpdateBoundLeft byteVals byteCursor - | Right -> queryUpdateBoundRight byteVals byteCursor curInput.MaxLen + | Right -> queryUpdateBoundRight byteVals byteCursor MAX_INPUT_LEN /// Get adjacent concrete byte values, toward the given direction. let queryNeighborBytes seed direction = - let curInput = getCurInput seed - let byteVals = curInput.ByteVals - let byteCursor = curInput.ByteCursor + let byteVals = seed.ByteVals + let byteCursor = seed.CursorPos match direction with | Stay -> failwith "queryNeighborBytes() cannot be called with 'Stay'" | Right -> @@ -172,57 +135,24 @@ module Seed = (************************ Content update functions ************************) - /// Update (replace) the current input sequence pointed by cursor. - let setCurInputSeq seed inputSeq = - match seed.SourceCursor with - | Args -> { seed with Args = inputSeq } - | File -> { seed with File = { seed.File with Content = inputSeq } } - | StdIn -> { seed with StdIn = inputSeq } - - /// Update (replace) the current input pointed by cursor. - let setCurInput seed input = - let curInputSeq = getCurInputSeq seed - let newInputSeq = InputSeq.setCurInput curInputSeq input - setCurInputSeq seed newInputSeq - - /// Update argument input sequence with given command-line argument string. - /// Note that maximum length of each input is set with the length of each - /// argument string. - let setArgs seed (cmdLine: string) = - let delim = [| ' '; '\t'; '\n' |] - let initArgs = cmdLine.Split(delim, StringSplitOptions.RemoveEmptyEntries) - let argInputs = - Array.map strToBytes initArgs - |> Array.filter (fun arg -> Array.length arg <> 0) - |> Array.map (fun bytes -> Input.makeWith bytes (Array.length bytes)) - let argInputSeq = { Inputs = argInputs; InputCursor = 0; CursorDir = Right } - {seed with Seed.Args = argInputSeq } - - /// Update input file path with the given string. - let setFilepath seed filepath = - { seed with File = { seed.File with Path = ConstPath filepath } } - /// Impose a constraint with lower and upper bounds, on the ByteVal at the /// given offset. let constrainByteAt seed direction offset low upper = - let curInput = getCurInput seed let byteCursor = match direction with | Stay -> failwith "constrainByteAt() cannot be called with 'Stay'" - | Right -> curInput.ByteCursor + offset - | Left -> curInput.ByteCursor - offset - let newByteVals = Array.copy curInput.ByteVals + | Right -> seed.CursorPos + offset + | Left -> seed.CursorPos - offset + let newByteVals = Array.copy seed.ByteVals let newByteVal = if low <> upper then Interval (low, upper) else Fixed low newByteVals.[byteCursor] <- newByteVal - let newInput = { curInput with ByteVals = newByteVals } - setCurInput seed newInput + { seed with ByteVals = newByteVals } /// Fix the current ByteVals pointed by the cursor, with the provided bytes. let fixCurBytes seed dir bytes = let nBytes = Array.length bytes - let curInput = getCurInput seed - let curByteVals = curInput.ByteVals - let byteCursor = curInput.ByteCursor + let curByteVals = seed.ByteVals + let byteCursor = seed.CursorPos let startPos = if dir = Right then byteCursor else byteCursor - nBytes + 1 let newByteVals = // Note that 'MaxLen' is already checked in queryUpdateBound(). @@ -231,229 +161,164 @@ module Seed = Array.append curByteVals (Array.init reqSize (fun _ -> Undecided 0uy)) else Array.copy curByteVals Array.iteri (fun i b -> newByteVals.[startPos + i] <- Fixed b) bytes - let newInput = { curInput with ByteVals = newByteVals } - setCurInput seed newInput + { seed with ByteVals = newByteVals } /// Update the current ByteVal pointed by the cursor. let updateCurByte seed byteVal = - let curInput = getCurInput seed - let curByteVals = curInput.ByteVals - let byteCursor = curInput.ByteCursor + let curByteVals = seed.ByteVals + let byteCursor = seed.CursorPos let newByteVals = Array.copy curByteVals newByteVals.[byteCursor] <- byteVal - let newInput = { curInput with ByteVals = newByteVals } - setCurInput seed newInput + { seed with ByteVals = newByteVals } - /// Update the ByteVal at given offset of current input. + /// Update the ByteVal at given offset of the seed. let updateByteValAt seed pos byteVal = - let curInput = getCurInput seed - let curByteVals = curInput.ByteVals + let curByteVals = seed.ByteVals let newByteVals = Array.copy curByteVals newByteVals.[pos] <- byteVal - let newInput = { curInput with ByteVals = newByteVals } - setCurInput seed newInput + { seed with ByteVals = newByteVals } - /// Update the bytes of current input, starting from the given offset. + /// Update the bytes of the seed, starting from the given offset. /// Approximate path conditions of the updated ByteVals are abandoned. let updateBytesFrom seed pos bytes = - let curInput = getCurInput seed - let curByteVals = curInput.ByteVals + let curByteVals = seed.ByteVals let newByteVals = Array.copy curByteVals Array.iteri (fun i b -> newByteVals.[pos + i] <- Undecided b) bytes - let newInput = { curInput with ByteVals = newByteVals } - setCurInput seed newInput + { seed with ByteVals = newByteVals } - /// Flip the bit at the given byte/bit offset of current input. + /// Flip the bit at the given byte/bit offset of the seed. let flipBitAt seed bytePos bitPos = - let curInput = getCurInput seed - let curByteVals = curInput.ByteVals + let curByteVals = seed.ByteVals let newByteVals = Array.copy curByteVals let curByteVal = curByteVals.[bytePos] let curByte = ByteVal.getConcreteByte curByteVal let newByte = curByte ^^^ ((byte 1) <<< bitPos) newByteVals.[bytePos] <- Undecided newByte - let newInput = { curInput with ByteVals = newByteVals } - setCurInput seed newInput + { seed with ByteVals = newByteVals } - /// Insert bytes at the given offset of the current input. + /// Insert bytes at the given offset of the seed. let insertBytesInto seed pos bytes = - let curInput = getCurInput seed - let curByteVals = curInput.ByteVals + let curByteVals = seed.ByteVals let curBytesLen = Array.length curByteVals let headByteVals = curByteVals.[0 .. (pos - 1)] let tailByteVals = curByteVals.[pos .. (curBytesLen - 1)] let byteVals = Array.map (fun b -> Undecided b) bytes let newByteVals = Array.concat [headByteVals; byteVals; tailByteVals] - let newByteVals = if Array.length newByteVals > curInput.MaxLen - then newByteVals.[.. (curInput.MaxLen - 1)] + let newByteVals = if Array.length newByteVals > MAX_INPUT_LEN + then newByteVals.[.. (MAX_INPUT_LEN - 1)] else newByteVals - let newInput = { curInput with ByteVals = newByteVals } - setCurInput seed newInput + { seed with ByteVals = newByteVals } - /// Remove bytes starting from the given offset of the current input. + /// Remove bytes starting from the given offset of the seed. let removeBytesFrom seed pos n = - let curInput = getCurInput seed - let curByteVals = curInput.ByteVals + let curByteVals = seed.ByteVals let curBytesLen = Array.length curByteVals let headByteVals = curByteVals.[0 .. (pos - 1)] let tailByteVals = curByteVals.[(pos + n) .. (curBytesLen - 1)] let newByteVals = Array.append headByteVals tailByteVals - let newInput = { curInput with ByteVals = newByteVals } - setCurInput seed newInput + { seed with ByteVals = newByteVals } (************************* Cursor update functions *************************) + /// Set the byte cursor position of the seed. + let setCursorPos seed newPos = { seed with CursorPos = newPos } + + /// Set the byte cursor direction of the seed. + let setCursorDir seed dir = { seed with CursorDir = dir } + + /// Step the byte cursor, following the cursor direction. + let stepCursor seed = + let byteCursor = seed.CursorPos + match seed.CursorDir with + | Stay -> None + | Left when 0 <= byteCursor - 1 -> + Some (setCursorPos seed (byteCursor - 1)) + | Right when byteCursor + 1 < seed.ByteVals.Length -> + Some (setCursorPos seed (byteCursor + 1)) + | Left _ | Right _ -> None + + // Starting from 'curIdx', find the index of the first unfixed ByteVal. + let rec private findUnfixedByte (bytes: ByteVal []) curIdx = + if curIdx < 0 || curIdx >= bytes.Length then -1 + elif ByteVal.isUnfixed bytes.[curIdx] then curIdx + else findUnfixedByte bytes (curIdx + 1) + + // Starting from 'curIdx', find the index of the first unfixed ByteVal, in a + // backward direction. + let rec private findUnfixedByteBackward (bytes: ByteVal []) curIdx = + if curIdx < 0 || curIdx >= bytes.Length then -1 + elif ByteVal.isUnfixed bytes.[curIdx] then curIdx + else findUnfixedByteBackward bytes (curIdx - 1) + + /// Move the byte cursor to an unfixed ByteVal. Cursor may stay at the same + /// position. + let private moveToUnfixedByte seed = + let byteCursor = seed.CursorPos + let cursorDir = seed.CursorDir + match cursorDir with + | Stay -> None + | Left -> let offset = findUnfixedByteBackward seed.ByteVals byteCursor + if offset <> -1 then Some (setCursorPos seed offset) else None + | Right -> let offset = findUnfixedByte seed.ByteVals byteCursor + if offset <> -1 then Some (setCursorPos seed offset) else None + + /// Move the byte cursor to the next unfixed ByteVal. Cursor should move at + /// least one offset toward the current cursor direction. + let proceedCursor seed = + match stepCursor seed with + | None -> None + | Some newSeed -> moveToUnfixedByte newSeed + /// Update the byte cursor direction of current input. let setByteCursorDir seed dir = - let curInput = getCurInput seed - let newInput = Input.setCursorDir curInput dir - setCurInput seed newInput - - /// Update the input cursor direction of current input sequence. - let setInputCursorDir seed dir = - let curInputSeq = getCurInputSeq seed - let newInputSeq = InputSeq.setCursorDir curInputSeq dir - setCurInputSeq seed newInputSeq - - /// Update the input source cursor of the given seed. - let setSourceCursor seed newInputSrc = - { seed with SourceCursor = newInputSrc } + { seed with CursorDir = dir } /// Randomly move byte cursor position within current input. let shuffleByteCursor seed = - let curInput = getCurInput seed - let curInputLen = curInput.ByteVals.Length - let newByteCursor = random.Next(curInputLen) - let newCursorDir = if newByteCursor > (curInputLen / 2) then Left else Right - let newInput = Input.setCursor curInput newByteCursor - let newInput = Input.setCursorDir newInput newCursorDir - setCurInput seed newInput - - /// Step the byte cursor within the current input, following the cursor - /// direction. - let stepByteCursor seed = - let curInput = getCurInput seed - match Input.stepCursor curInput with - | None -> None - | Some newInput -> Some (setCurInput seed newInput) - - /// Move the byte cursor within current input, to the next unfixed ByteVal. - let moveToUnfixedByte seed = - let curInput = getCurInput seed - match Input.moveToUnfixedByte curInput with - | None -> None - | Some newInput -> Some (setCurInput seed newInput) - - /// Proceed the byte cursor within the current input, to the next unfixed - /// ByteVal, following the current cursor direction. - let proceedByteCursor seed = - let curInput = getCurInput seed - match Input.proceedCursor curInput with - | None -> None - | Some newInput -> Some (setCurInput seed newInput) - - /// Randomly move input cursor within current input sequence pointed by the - /// source cursor. - let shuffleInputCursor seed = - let curInputSeq = getCurInputSeq seed - if InputSeq.isSingular curInputSeq then seed - else let curInputSeqLen = curInputSeq.Inputs.Length - let newInputCursor = random.Next(curInputSeqLen) - let newInputSeq = InputSeq.setCursor curInputSeq newInputCursor - setCurInputSeq seed newInputSeq - - /// Step the input cursor, following the cursor direction. - let stepInputCursor seed = - let curInputSeq = getCurInputSeq seed - match InputSeq.stepCursor curInputSeq with - | None -> None - | Some newInputSeq -> Some (setCurInputSeq seed newInputSeq) - - /// Within the current input sequence, find an input that has any unfixed - /// ByteVal, and return a new input sequence with updated input/byte cursors. - /// The cursor may stay at the same position. - let moveToUnfixedInput seed = - let curInputSeq = getCurInputSeq seed - match InputSeq.moveToUnfixedInput curInputSeq with - | None -> None - | Some newInputSeq -> Some (setCurInputSeq seed newInputSeq) - - /// Proceed input cursor to the next unfixed input, following the current - /// cursor direction. - let proceedInputCursor seed = - let curInputSeq = getCurInputSeq seed - match InputSeq.proceedCursor curInputSeq with - | None -> None - | Some newInputSeq -> Some (setCurInputSeq seed newInputSeq) - - /// Proceed input cursor to the next unfixed input, and byte cursor toe the - /// next unfixed ByteVal, following the current cursor directions. - let proceedCursors seed = - List.choose identity [ proceedByteCursor seed; proceedInputCursor seed ] + let curLength = seed.ByteVals.Length + let newByteCursor = random.Next(curLength) + let newCursorDir = if newByteCursor > (curLength / 2) then Left else Right + { seed with CursorPos = newByteCursor; CursorDir = newCursorDir} /// Return seeds with byte cursor updated to point unfixed ByteVal. Probing /// should occur in both left side and right side. - let moveByteCursor isFromConcolic seed = - let curInput = getCurInput seed - let curByteVal = Input.getCurByteVal curInput + let relocateCursor isFromConcolic seed = + let curByteVal = getCurByteVal seed let leftwardSeed = setByteCursorDir seed Left let leftwardSeeds = // Avoid sampling at the same offset. if isFromConcolic && ByteVal.isSampledByte curByteVal then - match stepByteCursor leftwardSeed with + match stepCursor leftwardSeed with | None -> [] | Some s -> [ s ] else [ leftwardSeed ] let rightwardSeed = setByteCursorDir seed Right let rightwardSeeds = - match stepByteCursor rightwardSeed with + match stepCursor rightwardSeed with | None -> [] | Some s -> [ s ] List.choose moveToUnfixedByte (leftwardSeeds @ rightwardSeeds) - |> List.map (fun seed -> setInputCursorDir seed Stay) - - /// Return seeds with field cursor updated to point unfixed field. Probing - /// should occur in both left side and right side. - let moveInputCursor seed = - List.map (setInputCursorDir seed) [Left; Right] - |> List.choose stepInputCursor - |> List.choose moveToUnfixedInput - - // Auxiliary function for moveSourceCursor(). - let private moveSourceCursorAux seed inputSrc = - match inputSrc with - | InputSrc.StdInput when seed.SourceCursor <> StdIn && - not (InputSeq.isEmpty seed.StdIn) -> - moveToUnfixedInput (setSourceCursor seed StdIn) - | InputSrc.FileInput argIdx when seed.SourceCursor <> File && - not (InputSeq.isEmpty seed.File.Content) -> - let newFile = { seed.File with Path = ArgIdx argIdx } - let newSeed = { seed with File = newFile } - moveToUnfixedInput (setSourceCursor newSeed File) - | _ -> None // If inputSrc is equal to current input source, return nothing - - /// Return seeds with source cursor updated to point new input sources, using - /// the provided syscall trace information. - let moveSourceCursor seed inputSrcs = - List.choose (moveSourceCursorAux seed) (Set.toList inputSrcs) - - /// Return seeds with moved byte cursors, input cursors, and source cursors. - /// This is equivalent to applying moveByteCursor(), moveInputCursor(), and - /// moveSourceCursor() at once. - let moveCursors seed isFromConcolic inputSrcs = - let byteCursorMoved = moveByteCursor isFromConcolic seed - let inputCursorMoved = moveInputCursor seed - let sourceCursorMoved = moveSourceCursor seed inputSrcs - byteCursorMoved @ inputCursorMoved @ sourceCursorMoved - - (*************************** Stringfy functions ***************************) - - let private argToStr args = - let argInputToStr argInput = sprintf "\"%s\"" (Input.concretizeArg argInput) - let argStrs = Array.map argInputToStr args.Inputs - String.concat " " argStrs + // Auxiliary function for byteValsToStr() that handles 'Untouched' ByteVals. + let private untouchedToStr untouchedList = + if List.isEmpty untouchedList then "" + elif List.length untouchedList < 4 then + " " + String.concat " " (List.map ByteVal.toString untouchedList) + else sprintf " ...%dbytes..." (List.length untouchedList) + + // Stringfy ByteVal list. + let rec private byteValsToStr accumUntouched accumStrs byteVals = + match byteVals with + | [] -> accumStrs + untouchedToStr (List.rev accumUntouched) + | headByteVal :: tailByteVals -> + (match headByteVal with + | Untouched _ -> // Just accumulate to 'accumUntouched' and continue + byteValsToStr (headByteVal :: accumUntouched) accumStrs tailByteVals + | Undecided _ | Fixed _ | Interval _ | Sampled _ -> + let untouchedStr = untouchedToStr (List.rev accumUntouched) + let byteValStr = ByteVal.toString headByteVal + let accumStrs = accumStrs + untouchedStr + " " + byteValStr + byteValsToStr [] accumStrs tailByteVals) // reset accumUntouched to [] + + /// Stringfy the given seed. let toString seed = - let argStr = argToStr seed.Args - let curInput = getCurInput seed - let inputSrc = seed.SourceCursor - let inputCursor = if inputSrc = Args then seed.Args.InputCursor else 0 - let inputStr = Input.toString curInput - sprintf "(%s) %A[%d]=(%s)" argStr inputSrc inputCursor inputStr + let byteVals = List.ofArray seed.ByteVals + let byteStr = byteValsToStr [] "" byteVals + sprintf "%s (%d) (%A)" byteStr seed.CursorPos seed.CursorDir diff --git a/src/Core/TestCase.fs b/src/Core/TestCase.fs deleted file mode 100644 index 21e76dc..0000000 --- a/src/Core/TestCase.fs +++ /dev/null @@ -1,42 +0,0 @@ -module Eclipser.TestCase - -open System -open FSharp.Data -open FSharp.Data.JsonExtensions -open Utils - -/// Represents a concrete test case that can be executed by the target program. -/// Note that 'TestCase' does not have additional information contained in -/// 'Seed' type (e.g. cursors, approximate path condition, etc). -type TestCase = { - Args : string array; - StdIn : byte array; - FilePath : string - FileContent : byte array; -} - -/// Concretize a Seed into a TestCase. -let fromSeed (seed: Seed) = - { TestCase.Args = InputSeq.concretizeArg seed.Args - TestCase.StdIn = Array.concat (InputSeq.concretize seed.StdIn) - TestCase.FilePath = Seed.concretizeFilepath seed - TestCase.FileContent = Array.concat (InputSeq.concretize seed.File.Content) } - -/// Convert a TestCase into a JSON string. -let toJSON (tc: TestCase) = - let args = Array.map escapeString tc.Args - |> Array.map (fun arg -> "\"" + arg + "\"") - |> String.concat "," - let stdin = Convert.ToBase64String tc.StdIn - let fileCont = Convert.ToBase64String tc.FileContent - let filepath = escapeString tc.FilePath - sprintf """{"args":[%s], "filepath":"%s","filecontent":"%s","stdin":"%s"}""" - args filepath fileCont stdin - -/// Convert a JSON string into a TestCase. -let fromJSON (str: string) = - let info = JsonValue.Parse(str) - { TestCase.Args = [| for v in info?args -> v.AsString() |] - StdIn = info?stdin.AsString() |> Convert.FromBase64String - FilePath = info?filepath.AsString() - FileContent = info?filecontent.AsString() |> Convert.FromBase64String } diff --git a/src/Core/Typedef.fs b/src/Core/Typedef.fs index a920ce1..a37619c 100644 --- a/src/Core/Typedef.fs +++ b/src/Core/Typedef.fs @@ -1,7 +1,5 @@ namespace Eclipser -open Utils - type Hash = uint64 type Signedness = Signed | Unsigned @@ -21,51 +19,16 @@ module Arch = | "x64" -> X64 | _ -> raise UnsupportedArchException -/// Fuzzing modes that specify input source to fuzz. -type FuzzMode = - /// Fuzz command line argument input source. - | ArgFuzz - /// Fuzz file input source. - | FileFuzz +/// Specifies which input source to fuzz. +type InputSource = /// Fuzz standard input source. - | StdinFuzz - /// Start by fuzzing command line argument, and then automatically identify - /// and fuzz standard/file input source. - | AutoFuzz - -module FuzzMode = - - /// Convert a string into FuzzMode. - let ofString (str: string) = - match str.ToLower() with - | "arg" -> ArgFuzz - | "stdin" -> StdinFuzz - | "file" -> FileFuzz - | "auto" -> AutoFuzz - | _ -> printLine "Supported input sources : 'arg', 'stdin', 'file', 'auto'." - exit 1 + | StdInput + /// Fuzz file input source. + | FileInput of filepath: string /// Direction that the cursor of a 'Seed' should move toward. type Direction = Stay | Left | Right -/// Type of input source. Currently we support three input sources (argument, -/// file, standard input). -type InputKind = Args | File | StdIn - -module InputKind = - /// For the given fuzzing mode, decide the initial input source to start with. - let ofFuzzMode = function - | ArgFuzz -> Args - | StdinFuzz -> StdIn - | FileFuzz -> File - | AutoFuzz -> Args - - /// Decides whether a sequence of multiple inputs is allowed. - let isSingularInput = function - | Args -> false - | StdIn -> true // Currently, consider only one-time standard input. - | File -> true // Currently, consider only one file input. - /// Priority of found seed. A seed that increased edge coverage is assigned /// 'Favored' priority, while a seed that increased path coverage is assigned /// 'Normal' priority. diff --git a/src/Core/Utils.fs b/src/Core/Utils.fs index a5fda5f..d0566e1 100644 --- a/src/Core/Utils.fs +++ b/src/Core/Utils.fs @@ -109,14 +109,15 @@ let assertFileExists file = let removeFile file = try System.IO.File.Delete(file) with _ -> () -/// Remove a list of files, without throwing exception. -let removeFiles files = - List.iter removeFile files - /// Remove a directory, without throwing exception. let removeDir dir = try System.IO.Directory.Delete(dir, recursive = true) with _ -> () +/// Write a file, without throwing exception. +let writeFile filePath content = + try System.IO.File.WriteAllBytes(filePath, content) with + | _ -> log "[Warning] Failed to write file '%s'" filePath + /// Create a directory if not exists. let createDirectoryIfNotExists dir = if not (System.IO.Directory.Exists dir) then diff --git a/src/Eclipser.fsproj b/src/Eclipser.fsproj index a0ccab7..10958e1 100644 --- a/src/Eclipser.fsproj +++ b/src/Eclipser.fsproj @@ -13,10 +13,8 @@ - - @@ -33,9 +31,6 @@ - - - diff --git a/src/Executor/Executor.fs b/src/Executor/Executor.fs index a180599..5be5d54 100644 --- a/src/Executor/Executor.fs +++ b/src/Executor/Executor.fs @@ -6,11 +6,11 @@ open System.Runtime.InteropServices open Config open Utils open Options -open Syscall -open TestCase + +let private WHITES = [| ' '; '\t'; '\n' |] /// Kinds of QEMU instrumentor. Each instrumentor serves different purposes. -type Tracer = Coverage | BranchAt | BranchAll | Syscall | BBCount +type Tracer = Coverage | BranchAt | BranchAll | BBCount /// Specifies the method to handle execution timeout. It is necessary to use GDB /// when replaying test cases against a gcov-compiled binary, to log coverage @@ -50,8 +50,6 @@ let coverageTracerX86 = sprintf "%s/qemu-trace-pathcov-x86" buildDir let coverageTracerX64 = sprintf "%s/qemu-trace-pathcov-x64" buildDir let branchTracerX86 = sprintf "%s/qemu-trace-feedback-x86" buildDir let branchTracerX64 = sprintf "%s/qemu-trace-feedback-x64" buildDir -let syscallTracerX86 = sprintf "%s/qemu-trace-syscall-x86" buildDir -let syscallTracerX64 = sprintf "%s/qemu-trace-syscall-x64" buildDir let bbCountTracerX86 = sprintf "%s/qemu-trace-bbcount-x86" buildDir let bbCountTracerX64 = sprintf "%s/qemu-trace-bbcount-x64" buildDir @@ -63,54 +61,65 @@ let selectTracer tracer arch = | BranchAt, X64 -> branchTracerX64 | BranchAll, X86 -> branchTracerX86 | BranchAll, X64 -> branchTracerX64 - | Syscall, X86 -> syscallTracerX86 - | Syscall, X64 -> syscallTracerX64 | BBCount, X86 -> bbCountTracerX86 | BBCount, X64 -> bbCountTracerX64 let mutable pathHashLog = "" let mutable branchTraceLog = "" let mutable coverageLog = "" -let mutable syscallTraceLog = "" let mutable dbgLog = "" +let mutable forkServerEnabled = false -// TODO : Reorder functions and create a top-level function -let cleanUpSharedMem () = - if release_shared_mem() < 0 then log "Failed to release shared memory!" - -let prepareSharedMem () = - let shmID = prepare_shared_mem() - if shmID < 0 then failwith "Failed to allcate shared memory" - set_env("CK_SHM_ID", string shmID) - -let initialize outdir verbosity = - pathHashLog <- System.IO.Path.Combine(outdir, ".path_hash") - branchTraceLog <- System.IO.Path.Combine(outdir, ".branch_trace") - coverageLog <- System.IO.Path.Combine(outdir, ".coverage") - syscallTraceLog <- System.IO.Path.Combine(outdir, ".syscall_trace") - dbgLog <- System.IO.Path.Combine(outdir, ".debug_msg") +let initialize opt = + let outDir = opt.OutDir + let verbosity = opt.Verbosity + // Set environment variables for the instrumentor. + pathHashLog <- System.IO.Path.Combine(outDir, ".path_hash") + branchTraceLog <- System.IO.Path.Combine(outDir, ".branch_trace") + coverageLog <- System.IO.Path.Combine(outDir, ".coverage") + dbgLog <- System.IO.Path.Combine(outDir, ".debug_msg") set_env("CK_HASH_LOG", System.IO.Path.GetFullPath(pathHashLog)) set_env("CK_FEED_LOG", System.IO.Path.GetFullPath(branchTraceLog)) set_env("CK_COVERAGE_LOG", System.IO.Path.GetFullPath(coverageLog)) - set_env("CK_SYSCALL_LOG", System.IO.Path.GetFullPath(syscallTraceLog)) if verbosity >= 2 then set_env("CK_DBG_LOG", System.IO.Path.GetFullPath(dbgLog)) - // Set all the other env. variables in advance, to avoid affecting path hash. + // Set other environment variables in advance, to avoid affecting path hash. set_env("CK_MODE", "0") set_env("CK_FEED_ADDR", "0") set_env("CK_FEED_IDX", "0") - set_env("CK_FORK_SERVER", "0") // In default, fork server is not enabled. set_env("CK_CTX_SENSITIVITY", string CtxSensitivity) + initialize_exec TimeoutHandling.SendSigterm + // Initialize shared memory. + let shmID = prepare_shared_mem() + if shmID < 0 then failwith "Failed to allcate shared memory" + set_env("CK_SHM_ID", string shmID) + // Initialize fork server. + forkServerEnabled <- true + set_env("CK_FORK_SERVER", "1") + let cmdLine = opt.Arg.Split(WHITES, StringSplitOptions.RemoveEmptyEntries) + let coverageTracer = selectTracer Coverage opt.Architecture + let args = Array.append [|coverageTracer; opt.TargetProg|] cmdLine + let pidCoverage = init_forkserver_coverage(args.Length, args, opt.ExecTimeout) + if pidCoverage = -1 then + failwith "Failed to initialize fork server for coverage tracer" + let branchTracer = selectTracer BranchAll opt.Architecture + let args = Array.append [|branchTracer; opt.TargetProg|] cmdLine + let pidBranch = init_forkserver_branch (args.Length, args, opt.ExecTimeout) + if pidBranch = -1 then + failwith "Failed to initialize fork server for branch tracer" -let cleanUpFiles () = - removeFiles [pathHashLog; branchTraceLog; coverageLog] - removeFiles [syscallTraceLog; dbgLog] +let cleanup () = + if forkServerEnabled then kill_forkserver () + if release_shared_mem() < 0 then log "Failed to release shared memory!" + removeFile pathHashLog + removeFile branchTraceLog + removeFile coverageLog + removeFile dbgLog (*** Statistics ***) let mutable totalExecutions = 0 let mutable phaseExecutions = 0 - let getTotalExecutions () = totalExecutions let getPhaseExecutions () = phaseExecutions let resetPhaseExecutions () = phaseExecutions <- 0 @@ -128,70 +137,32 @@ let incrExecutionCount tracerTyp = totalExecutions <- totalExecutions + 1 phaseExecutions <- phaseExecutions + 1 -(*** Fork server ***) - -let mutable forkServerEnabled = false - -let initForkServer opt = - forkServerEnabled <- true - set_env("CK_FORK_SERVER", "1") - let white = [| ' '; '\t'; '\n' |] - let initArgStr = opt.InitArg - let initArgs = initArgStr.Split(white, StringSplitOptions.RemoveEmptyEntries) - let coverageTracer = selectTracer Coverage opt.Architecture - let args = Array.append [|coverageTracer; opt.TargetProg|] initArgs - let pidCoverage = init_forkserver_coverage(args.Length, args, opt.ExecTimeout) - if pidCoverage = -1 then - failwith "Failed to initialize fork server for coverage tracer" - let branchTracer = selectTracer BranchAll opt.Architecture - let args = Array.append [|branchTracer; opt.TargetProg|] initArgs - let pidBranch = init_forkserver_branch (args.Length, args, opt.ExecTimeout) - if pidBranch = -1 then - failwith "Failed to initialize fork server for branch tracer" - let abandonForkServer () = log "Abandon fork server" forkServerEnabled <- false set_env("CK_FORK_SERVER", "0") kill_forkserver () -let cleanUpForkServer () = - if forkServerEnabled then kill_forkserver () - (*** File handling utilities ***) -let touchFile filename = - // Explicitly delete file before creating one, to avoid hanging when the - // specified file already exists as a FIFO. - try System.IO.File.Delete(filename) - System.IO.File.WriteAllBytes(filename, [|0uy|]) - with _ -> () - -let writeFile filename content = - try System.IO.File.WriteAllBytes(filename, content) with - | _ -> log "[Warning] Failed to write file '%s'" filename - let readAllLines filename = try List.ofSeq (System.IO.File.ReadLines filename) with | :? System.IO.FileNotFoundException -> [] -let isProgPath targetProg argStr = - try Path.GetFullPath argStr = targetProg with - | _ -> false - -let setupFiles targetProg (tc: TestCase) autoFuzzMode = - let argFiles = - if autoFuzzMode then - // Create files with argument strings as name, to identify input sources. - List.ofArray tc.Args - |> List.filter (fun arg -> not (isProgPath targetProg arg)) - |> (fun args -> List.iter touchFile args; args) - else [] - let inputFile = tc.FilePath - if inputFile <> "" then - writeFile inputFile tc.FileContent - inputFile :: argFiles - else argFiles +let private setupFile seed = + match seed.Source with + | StdInput -> () + | FileInput filePath -> writeFile filePath (Seed.concretize seed) + +let private clearFile seed = + match seed.Source with + | StdInput -> () + | FileInput filePath -> removeFile filePath + +let private prepareStdIn seed = + match seed.Source with + | StdInput -> Seed.concretize seed + | FileInput filePath -> [| |] (*** Tracer result parsing functions ***) @@ -270,118 +241,109 @@ let private readBranchTrace opt filename tryVal = (*** Tracer execute functions ***) -let private runTracer (tc: TestCase) tracerType opt = +let private runTracer tracerType opt (stdin: byte array) = incrExecutionCount tracerType let targetProg = opt.TargetProg let timeout = opt.ExecTimeout let usePty = opt.UsePty let tracer = selectTracer tracerType opt.Architecture - let args = Array.append [|tracer; targetProg|] tc.Args + let cmdLine = opt.Arg.Split(WHITES, StringSplitOptions.RemoveEmptyEntries) + let args = Array.append [|tracer; targetProg|] cmdLine let argc = args.Length - exec(argc, args, tc.StdIn.Length, tc.StdIn, timeout, usePty) + exec(argc, args, stdin.Length, stdin, timeout, usePty) -let private runCoverageTracerForked (tc: TestCase) opt = +let private runCoverageTracerForked opt (stdin: byte array) = incrExecutionCount Coverage let timeout = opt.ExecTimeout - let signal = exec_fork_coverage(timeout, tc.StdIn.Length, tc.StdIn) + let signal = exec_fork_coverage(timeout, stdin.Length, stdin) if signal = Signal.ERROR then abandonForkServer () signal -let private runBranchTracerForked (tc: TestCase) tracerType opt = +let private runBranchTracerForked tracerType opt (stdin: byte array) = incrExecutionCount tracerType let timeout = opt.ExecTimeout - let signal = exec_fork_branch(timeout, tc.StdIn.Length, tc.StdIn) + let signal = exec_fork_branch(timeout, stdin.Length, stdin) if signal = Signal.ERROR then abandonForkServer () signal (*** Top-level tracer executor functions ***) let getEdgeHash opt seed = - let tc = TestCase.fromSeed seed - let autoFuzzMode = opt.FuzzMode = AutoFuzz - let files = setupFiles opt.TargetProg tc autoFuzzMode set_env("CK_MODE", string (int CoverageTracerMode.EdgeHash)) - let _ = if forkServerEnabled - then runCoverageTracerForked tc opt - else runTracer tc Coverage opt + setupFile seed + let stdin = prepareStdIn seed + if forkServerEnabled then runCoverageTracerForked opt stdin + else runTracer Coverage opt stdin + |> ignore let edgeHash = parseExecHash coverageLog - removeFiles files + clearFile seed edgeHash let getCoverage opt seed = - let tc = TestCase.fromSeed seed - let autoFuzzMode = opt.FuzzMode = AutoFuzz - let files = setupFiles opt.TargetProg tc autoFuzzMode set_env("CK_MODE", string (int CoverageTracerMode.CountNewEdge)) + setupFile seed + let stdin = prepareStdIn seed let exitSig = if forkServerEnabled - then runCoverageTracerForked tc opt - else runTracer tc Coverage opt + then runCoverageTracerForked opt stdin + else runTracer Coverage opt stdin let newEdgeCnt, pathHash, edgeHash = parseCoverage coverageLog - removeFiles files + clearFile seed (newEdgeCnt, pathHash, edgeHash, exitSig) let getEdgeSet opt seed = - let tc = TestCase.fromSeed seed - let autoFuzzMode = opt.FuzzMode = AutoFuzz - let files = setupFiles opt.TargetProg tc autoFuzzMode set_env("CK_MODE", string (int CoverageTracerMode.EdgeSet)) - let _ = if forkServerEnabled - then runCoverageTracerForked tc opt - else runTracer tc Coverage opt + setupFile seed + let stdin = prepareStdIn seed + if forkServerEnabled then runCoverageTracerForked opt stdin + else runTracer Coverage opt stdin + |> ignore let edgeSet = readEdgeSet opt coverageLog - removeFiles files + clearFile seed edgeSet let getBranchTrace opt seed tryVal = - let tc = TestCase.fromSeed seed set_env("CK_FEED_ADDR", "0") set_env("CK_FEED_IDX", "0") - let autoFuzzMode = opt.FuzzMode = AutoFuzz - let files = setupFiles opt.TargetProg tc autoFuzzMode - let _ = if forkServerEnabled - then runBranchTracerForked tc BranchAll opt - else runTracer tc BranchAll opt + setupFile seed + let stdin = prepareStdIn seed + if forkServerEnabled then runBranchTracerForked BranchAll opt stdin + else runTracer BranchAll opt stdin + |> ignore let pathHash = parseExecHash pathHashLog let branchTrace = readBranchTrace opt branchTraceLog tryVal - removeFiles (pathHashLog :: branchTraceLog :: files) + clearFile seed + removeFile pathHashLog + removeFile branchTraceLog pathHash, branchTrace let getBranchInfoAt opt seed tryVal targPoint = - let tc = TestCase.fromSeed seed set_env("CK_FEED_ADDR", sprintf "%016x" targPoint.Addr) set_env("CK_FEED_IDX", sprintf "%016x" targPoint.Idx) - let autoFuzzMode = opt.FuzzMode = AutoFuzz - let files = setupFiles opt.TargetProg tc autoFuzzMode - let _ = if forkServerEnabled - then runBranchTracerForked tc BranchAt opt - else runTracer tc BranchAt opt + setupFile seed + let stdin = prepareStdIn seed + if forkServerEnabled then runBranchTracerForked BranchAt opt stdin + else runTracer BranchAt opt stdin + |> ignore let pathHash = parseExecHash pathHashLog let branchInfoOpt = match readBranchTrace opt branchTraceLog tryVal with | [] -> None | [ branchInfo ] -> Some branchInfo | branchInfos -> None - removeFiles (pathHashLog :: branchTraceLog :: files) + clearFile seed + removeFile pathHashLog + removeFile branchTraceLog (pathHash, branchInfoOpt) -let getSyscallTrace opt seed = - let tc = TestCase.fromSeed seed - let autoFuzzMode = opt.FuzzMode = AutoFuzz - let files = setupFiles opt.TargetProg tc autoFuzzMode - let _ = runTracer tc Syscall opt - let syscallTrace = readAllLines syscallTraceLog - let inputSrcs = checkInputSource opt.TargetProg tc.Args syscallTrace - removeFiles (syscallTraceLog :: files) - inputSrcs - -let nativeExecute opt tc = +let nativeExecute opt seed = let targetProg = opt.TargetProg - let autoFuzzMode = opt.FuzzMode = AutoFuzz - let files = setupFiles opt.TargetProg tc autoFuzzMode + setupFile seed + let stdin = prepareStdIn seed let timeout = opt.ExecTimeout let usePty = opt.UsePty - let args = Array.append [| targetProg |] tc.Args + let cmdLine = opt.Arg.Split(WHITES, StringSplitOptions.RemoveEmptyEntries) + let args = Array.append [| targetProg |] cmdLine let argc = args.Length - let signal = exec(argc, args, tc.StdIn.Length, tc.StdIn, timeout, usePty) - removeFiles files + let signal = exec(argc, args, stdin.Length, stdin, timeout, usePty) + clearFile seed signal diff --git a/src/Fuzz/Fuzz.fs b/src/Fuzz/Fuzz.fs index 014566c..643d0f3 100644 --- a/src/Fuzz/Fuzz.fs +++ b/src/Fuzz/Fuzz.fs @@ -1,30 +1,22 @@ module Eclipser.Fuzz -open System open Config open Utils open Options -let findInputSrc opt seed = - if opt.FuzzMode = AutoFuzz - then Executor.getSyscallTrace opt seed - else Set.empty - -let rec moveCursorsAux opt isFromConcolic accConcolic accRandom items = +let rec makeItemsAux opt isFromConcolic accConcolic accRandom items = match items with | [] -> (accConcolic, accRandom) | (priority, seed) :: tailItems -> - let inputSrcs = findInputSrc opt seed - let concSeeds = Seed.moveCursors seed isFromConcolic inputSrcs - let concItems = List.map (fun s -> (priority, s)) concSeeds - let randSeeds = seed :: Seed.moveSourceCursor seed inputSrcs - let randItems = List.map (fun s -> (priority, s)) randSeeds + let concItems = Seed.relocateCursor isFromConcolic seed + |> List.map (fun s -> (priority, s)) + let randItems = [(priority, seed)] let accConcolic = concItems @ accConcolic let accRandom = randItems @ accRandom - moveCursorsAux opt isFromConcolic accConcolic accRandom tailItems + makeItemsAux opt isFromConcolic accConcolic accRandom tailItems -let moveCursors opt isFromConcolic seeds = - let concItems, randItems = moveCursorsAux opt isFromConcolic [] [] seeds +let makeItems opt isFromConcolic seeds = + let concItems, randItems = makeItemsAux opt isFromConcolic [] [] seeds (List.rev concItems, List.rev randItems) /// Allocate testing resource for each strategy (grey-box concolic testing and @@ -55,9 +47,12 @@ let rec greyConcolicLoop opt concQ randQ = log "Grey-box concolic on %A seed : %s" pr (Seed.toString seed) let newSeeds = GreyConcolic.run seed opt // Move cursors of newly generated seeds. - let newItemsForConc, newItemsForRand = moveCursors opt true newSeeds + let newItemsForConc, newItemsForRand = makeItems opt true newSeeds // Also generate seeds by just stepping the cursor of original seed. - let steppedItems = List.map (fun s -> (pr, s)) (Seed.proceedCursors seed) + let steppedItems = + match Seed.proceedCursor seed with + | None -> [] + | Some s -> [(pr, s)] let concQ = List.fold ConcolicQueue.enqueue concQ newItemsForConc let concQ = List.fold ConcolicQueue.enqueue concQ steppedItems // Note that 'Stepped' seeds are not enqueued for random fuzzing. @@ -86,7 +81,7 @@ let rec randFuzzLoop opt concQ randQ = log "Random fuzzing on %A seed : %s" pr (Seed.toString seed) let newSeeds = RandomFuzz.run seed opt // Move cursors of newly generated seeds. - let newItemsForConc, newItemsForRand = moveCursors opt false newSeeds + let newItemsForConc, newItemsForRand = makeItems opt false newSeeds let concQ = List.fold ConcolicQueue.enqueue concQ newItemsForConc let randQ = List.fold RandFuzzQueue.enqueue randQ newItemsForRand randFuzzLoop opt concQ randQ @@ -124,14 +119,13 @@ let fuzzingTimer timeoutSec queueDir = async { log "===== Statistics =====" Manager.printStatistics () log "Done, clean up and exit..." - Executor.cleanUpForkServer () - Executor.cleanUpSharedMem () - Executor.cleanUpFiles () + Executor.cleanup () removeDir queueDir exit (0) } -let run args = +[] +let main args = let opt = parseFuzzOption args validateFuzzOption opt assertFileExists opt.TargetProg @@ -139,13 +133,10 @@ let run args = log "[*] Time limit : %d sec" opt.Timelimit createDirectoryIfNotExists opt.OutDir Manager.initialize opt.OutDir - Executor.initialize opt.OutDir opt.Verbosity - Executor.initialize_exec Executor.TimeoutHandling.SendSigterm - Executor.prepareSharedMem () - if opt.FuzzMode = StdinFuzz || opt.FuzzMode = FileFuzz then - Executor.initForkServer opt + Executor.initialize opt let queueDir = sprintf "%s/.internal" opt.OutDir let greyConcQueue, randFuzzQueue = Initialize.initQueue opt queueDir log "[*] Fuzzing starts" Async.Start (fuzzingTimer opt.Timelimit queueDir) fuzzLoop opt greyConcQueue randFuzzQueue + 0 // Unreachable diff --git a/src/Fuzz/GreyConcolic/GreyConcolic.fs b/src/Fuzz/GreyConcolic/GreyConcolic.fs index f51d004..738f4c2 100644 --- a/src/Fuzz/GreyConcolic/GreyConcolic.fs +++ b/src/Fuzz/GreyConcolic/GreyConcolic.fs @@ -47,7 +47,7 @@ let checkByProducts opt spawnedSeeds = let run seed opt = let curByteVal = Seed.getCurByteVal seed - let minByte, maxByte = ByteVal.getMinMax curByteVal seed.SourceCursor + let minByte, maxByte = ByteVal.getMinMax curByteVal seed.Source if minByte = maxByte then let seedStr = Seed.toString seed failwithf "Cursor pointing to Fixed ByteVal %s" seedStr diff --git a/src/Fuzz/GreyConcolic/Solve.fs b/src/Fuzz/GreyConcolic/Solve.fs index d710cdb..0a28b18 100644 --- a/src/Fuzz/GreyConcolic/Solve.fs +++ b/src/Fuzz/GreyConcolic/Solve.fs @@ -129,14 +129,10 @@ module GreySolver = let solveMonotonic seed opt accRes (targPt, mono) = let maxLenR = Seed.queryUpdateBound seed Right let maxLenL = Seed.queryUpdateBound seed Left - match seed.SourceCursor with - | InputKind.Args -> - binarySearch seed opt Right maxLenR targPt accRes mono - | _ -> // Try big endian first, and stop if any result is found. - let res = binarySearch seed opt Right maxLenR targPt [] mono - if List.isEmpty res - then binarySearch seed opt Left maxLenL targPt accRes mono - else res @ accRes + // Try big endian first, and stop if any result is found. + let res = binarySearch seed opt Right maxLenR targPt [] mono + if not (List.isEmpty res) then res @ accRes + else binarySearch seed opt Left maxLenL targPt accRes mono let solveMonotonics seed opt monotonics = let solveN = opt.NSolve / 3 diff --git a/src/Fuzz/Initialize.fs b/src/Fuzz/Initialize.fs index 0725f13..155d364 100644 --- a/src/Fuzz/Initialize.fs +++ b/src/Fuzz/Initialize.fs @@ -1,76 +1,25 @@ module Eclipser.Initialize -open System -open Config open Utils open Options -let createSeeds opt = - let maxArgLens = List.filter (fun len -> len > 0) opt.MaxArgLen - let inputSrc = InputKind.ofFuzzMode opt.FuzzMode - let initSeedWithNArgs n = - let (maxArgLens', _) = splitList n maxArgLens - Seed.make inputSrc maxArgLens' opt.MaxFileLen opt.MaxStdInLen - List.ofSeq { 1 .. List.length maxArgLens } - |> List.map initSeedWithNArgs - |> List.rev // Prioritize seed with more # of args, for better exploration. +let private initializeSeeds opt = + if opt.InputDir = "" then [Seed.make opt.FuzzSource] + else System.IO.Directory.EnumerateFiles opt.InputDir // Obtain file list + |> List.ofSeq // Convert list to array + |> List.map System.IO.File.ReadAllBytes // Read in file contents + |> List.map (Seed.makeWith opt.FuzzSource) // Create seed with content -let importSeeds opt = - let inputSrc = InputKind.ofFuzzMode opt.FuzzMode - let maxLen = - match inputSrc with - | Args when List.length opt.MaxArgLen = 1 -> List.head opt.MaxArgLen - | Args -> failwith "Invalid max length option on argument input" - | File -> opt.MaxFileLen - | StdIn -> opt.MaxStdInLen - System.IO.Directory.EnumerateFiles opt.InitSeedsDir // Obtain file list - |> List.ofSeq // Convert list to array - |> List.map System.IO.File.ReadAllBytes // Read in file contents - |> List.map (Seed.makeWith inputSrc maxLen) // Create seed with content - -let initializeSeeds opt = - let initArg = opt.InitArg - let fpath = opt.Filepath - let seedDir = opt.InitSeedsDir - // Initialize seeds, and set the initial argument/file path of each seed - if seedDir = "" then createSeeds opt else importSeeds opt - |> List.map (fun s -> if initArg <> "" then Seed.setArgs s initArg else s) - |> List.map (fun s -> if fpath <> "" then Seed.setFilepath s fpath else s) - -let findInputSrc opt seed = - if opt.FuzzMode = AutoFuzz - then Executor.getSyscallTrace opt seed - else Set.empty - -let rec moveCursorsAux opt isFromConcolic accConcolic accRandom items = - match items with - | [] -> (accConcolic, accRandom) - | (priority, seed) :: tailItems -> - let inputSrcs = findInputSrc opt seed - let concSeeds = Seed.moveCursors seed isFromConcolic inputSrcs - let concItems = List.map (fun s -> (priority, s)) concSeeds - let randSeeds = seed :: Seed.moveSourceCursor seed inputSrcs - let randItems = List.map (fun s -> (priority, s)) randSeeds - let accConcolic = concItems @ accConcolic - let accRandom = randItems @ accRandom - moveCursorsAux opt isFromConcolic accConcolic accRandom tailItems - -let moveCursors opt isFromConcolic seeds = - let concItems, randItems = moveCursorsAux opt isFromConcolic [] [] seeds - (List.rev concItems, List.rev randItems) - -let preprocessAux opt seed = +let private preprocessAux opt seed = let newEdgeN, pathHash, edgeHash, exitSig = Executor.getCoverage opt seed let isNewPath = Manager.save opt seed newEdgeN pathHash edgeHash exitSig true - let inputSrcs = findInputSrc opt seed - let newSeeds = seed :: Seed.moveSourceCursor seed inputSrcs - if newEdgeN > 0 then List.map (fun s -> (Favored, s)) newSeeds - elif isNewPath then List.map (fun s -> (Normal, s)) newSeeds - else [] + if newEdgeN > 0 then Some (Favored, seed) + elif isNewPath then Some (Normal, seed) + else None -let preprocess opt seeds = +let private preprocess opt seeds = log "[*] Total %d initial seeds" (List.length seeds) - let items = List.collect (preprocessAux opt) seeds + let items = List.choose (preprocessAux opt) seeds let favoredCount = List.filter (fst >> (=) Favored) items |> List.length let normalCount = List.filter (fst >> (=) Normal) items |> List.length log "[*] %d initial items with high priority" favoredCount diff --git a/src/Fuzz/Manager.fs b/src/Fuzz/Manager.fs index 8be3e0a..8889906 100644 --- a/src/Fuzz/Manager.fs +++ b/src/Fuzz/Manager.fs @@ -21,9 +21,6 @@ let mutable private segfaultCount = 0 let mutable private illegalInstrCount = 0 let mutable private fpErrorCount = 0 let mutable private abortCount = 0 -let mutable private argCrashCount = 0 -let mutable private stdinCrashCount = 0 -let mutable private fileCrashCount = 0 let mutable private crashCount = 0 let mutable private argTestCaseCount = 0 let mutable private stdinTestCaseCount = 0 @@ -59,10 +56,6 @@ let printStatistics () = log " Illegal instruction : %d" illegalInstrCount log " Floating point error : %d" fpErrorCount log " Program abortion : %d" abortCount - log "Input vector of crashes" - log " Argument : %d" argCrashCount - log " Stdin : %d" stdinCrashCount - log " File : %d" fileCrashCount let private updateCrashCount seed exitSig = match exitSig with @@ -71,50 +64,39 @@ let private updateCrashCount seed exitSig = | Signal.SIGFPE -> fpErrorCount <- fpErrorCount + 1 | Signal.SIGABRT -> abortCount <- abortCount + 1 | _ -> failwith "updateCrashCount() called with a non-crashing exit signal" - match seed.SourceCursor with - | Args -> argCrashCount <- argCrashCount + 1 - | StdIn -> stdinCrashCount <- stdinCrashCount + 1 - | File -> fileCrashCount <- fileCrashCount + 1 crashCount <- crashCount + 1 let private updateTestcaseCount seed = - match seed.SourceCursor with - | Args -> argTestCaseCount <- argTestCaseCount + 1 - | StdIn -> stdinTestCaseCount <- stdinTestCaseCount + 1 - | File -> fileTestCaseCount <- fileTestCaseCount + 1 testCaseCount <- testCaseCount + 1 (*** Test case storing functions ***) let private dumpCrash opt seed exitSig = if opt.Verbosity >= 0 then log "Save crash seed : %s" (Seed.toString seed) - let crashTC = TestCase.fromSeed seed - let crashName = sprintf "crash-%d" crashCount + let crashName = sprintf "crash-%05d" crashCount let crashPath = System.IO.Path.Combine(crashDir, crashName) - System.IO.File.WriteAllText(crashPath, (TestCase.toJSON crashTC)) + System.IO.File.WriteAllBytes(crashPath, Seed.concretize seed) updateCrashCount seed exitSig let private dumpTestCase seed = - let tc = TestCase.fromSeed seed - let tcName = sprintf "tc-%d" testCaseCount + let tcName = sprintf "tc-%05d" testCaseCount let tcPath = System.IO.Path.Combine(testcaseDir, tcName) - System.IO.File.WriteAllText(tcPath, (TestCase.toJSON tc)) + System.IO.File.WriteAllBytes(tcPath, Seed.concretize seed) updateTestcaseCount seed -let private checkCrash opt exitSig tc edgeHash = +let private checkCrash opt exitSig seed edgeHash = if Signal.isCrash exitSig && addCrashHash edgeHash then (true, exitSig) elif Signal.isTimeout exitSig then // Check again with native execution - let exitSig' = Executor.nativeExecute opt tc + let exitSig' = Executor.nativeExecute opt seed if Signal.isCrash exitSig' && addCrashHash edgeHash then (true, exitSig') else (false, exitSig') else (false, exitSig) let save opt seed newN pathHash edgeHash exitSig isInitSeed = - let tc = TestCase.fromSeed seed let isNewPath = addPathHash pathHash - let isNewCrash, exitSig' = checkCrash opt exitSig tc edgeHash + let isNewCrash, exitSig' = checkCrash opt exitSig seed edgeHash if newN > 0 || isInitSeed then dumpTestCase seed if isNewCrash then dumpCrash opt seed exitSig' isNewPath diff --git a/src/Fuzz/Options.fs b/src/Fuzz/Options.fs index 08a934b..82988a9 100644 --- a/src/Fuzz/Options.fs +++ b/src/Fuzz/Options.fs @@ -9,14 +9,10 @@ type FuzzerCLI = | [] [] Verbose of int | [] [] [] Timelimit of sec: int | [] [] [] OutputDir of path: string - | [] [] Src of string // Options related to seed initialization - | [] [] InitSeedsDir of path: string - | [] MaxArgLen of int list - | [] MaxFileLen of int - | [] MaxStdInLen of int - | [] InitArg of string - | [] [] [] Filepath of string + | [] [] InputDir of path: string + | [] Arg of string + | [] [] Filepath of string // Options related to execution of program | [] ExecTimeout of millisec:uint64 | [] UsePty @@ -34,16 +30,9 @@ with | Verbose _ -> "Verbosity level to control debug messages (default:0)." | Timelimit _ -> "Timeout for fuzz testing (in seconds)." | OutputDir _ -> "Directory to store testcase outputs." - | Src _ -> "Input source to fuzz ().\n" + - "In case of 'auto', fuzzer will start from fuzzing command " + - "line arguments and then automatically identify and fuzz " + - "standard input and file inputs." // Options related to seed initialization - | InitSeedsDir _ -> "Directory containing initial seeds." - | MaxArgLen _ -> "Maximum len of cmdline argument (default: [8byte])" - | MaxFileLen _ -> "Maximum len of file input (default: 1MB)" - | MaxStdInLen _ -> "Maximum len of standard input (default: 1MB)" - | InitArg _ -> "Initial command-line argument of program under test." + | InputDir _ -> "Directory containing initial seeds." + | Arg _ -> "Command-line argument of program under test." | Filepath _ -> "File input's (fixed) path" // Options related to execution of program | ExecTimeout _ -> "Execution timeout (ms) for a fuzz run (default:500)" @@ -62,19 +51,15 @@ type FuzzOption = { Verbosity : int OutDir : string Timelimit : int - FuzzMode : FuzzMode // Options related to program execution TargetProg : string ExecTimeout : uint64 UsePty : bool Architecture : Arch // Options related to seed - InitSeedsDir : string - MaxArgLen : int list - MaxFileLen : int - MaxStdInLen : int - InitArg : string - Filepath : string + FuzzSource : InputSource + InputDir : string + Arg : string // Options related to test case generation NSolve : int NSpawn : int @@ -90,7 +75,6 @@ let parseFuzzOption (args: string array) = { Verbosity = r.GetResult (<@ Verbose @>, defaultValue = 0) OutDir = r.GetResult (<@ OutputDir @>) Timelimit = r.GetResult (<@ Timelimit @>) - FuzzMode = FuzzMode.ofString (r.GetResult(<@ Src @>)) // Options related to program execution TargetProg = System.IO.Path.GetFullPath(r.GetResult (<@ Program @>)) ExecTimeout = r.GetResult (<@ ExecTimeout @>, defaultValue = DefaultExecTO) @@ -98,12 +82,10 @@ let parseFuzzOption (args: string array) = Architecture = r.GetResult(<@ Architecture @>, defaultValue = "X64") |> Arch.ofString // Options related to seed - InitSeedsDir = r.GetResult(<@ InitSeedsDir @>, defaultValue = "") - MaxArgLen = r.GetResult (<@ MaxArgLen @>, defaultValue = [8]) - MaxFileLen = r.GetResult (<@ MaxFileLen @>, defaultValue = 1048576) - MaxStdInLen = r.GetResult (<@ MaxStdInLen @>, defaultValue = 1048576) - InitArg = r.GetResult (<@ InitArg @>, defaultValue = "") - Filepath = r.GetResult (<@ Filepath @>, defaultValue = "") + FuzzSource = if not (r.Contains(<@ Filepath @>)) then StdInput + else FileInput (r.GetResult (<@ Filepath @>)) + InputDir = r.GetResult(<@ InputDir @>, defaultValue = "") + Arg = r.GetResult (<@ Arg @>, defaultValue = "") // Options related to test case generation NSolve = r.GetResult(<@ NSolve @>, defaultValue = 600) NSpawn = r.GetResult(<@ NSpawn @>, defaultValue = 10) @@ -111,9 +93,6 @@ let parseFuzzOption (args: string array) = RandFuzzOnly = r.Contains(<@ RandFuzzOnly @>) } let validateFuzzOption opt = - if opt.FuzzMode = FileFuzz && opt.Filepath = "" then - printLine "Should specify the file input path in file fuzzing mode." - exit 1 if opt.GreyConcolicOnly && opt.RandFuzzOnly then printLine "Cannot specify '--greyconcoliconly' and 'randfuzzonly' together." exit 1 diff --git a/src/Fuzz/RandomFuzz.fs b/src/Fuzz/RandomFuzz.fs index 509d1f7..f7d61b8 100644 --- a/src/Fuzz/RandomFuzz.fs +++ b/src/Fuzz/RandomFuzz.fs @@ -61,7 +61,7 @@ let roundUpExp input = let rec trimAux opt accSeed edgeHash trimMinSize trimSize pos = if trimSize < trimMinSize then accSeed // Cannot lower trimSize anymore, time to stop - elif pos + trimSize >= Seed.getCurInputLen accSeed then + elif pos + trimSize >= Seed.getCurLength accSeed then (* Reached end, retry with more fine granularity (reset idx to 0). *) trimAux opt accSeed edgeHash trimMinSize (trimSize / 2) 0 else let trySeed = Seed.removeBytesFrom accSeed pos trimSize @@ -74,7 +74,7 @@ let rec trimAux opt accSeed edgeHash trimMinSize trimSize pos = trimAux opt accSeed edgeHash trimMinSize trimSize newPos let trim opt edgeHash seed = - let inputLen = Seed.getCurInputLen seed + let inputLen = Seed.getCurLength seed let inputLenRounded = roundUpExp inputLen let trimSize = max (inputLenRounded / TRIM_START_STEPS) TRIM_MIN_BYTES let trimMinSize = max (inputLenRounded / TRIM_END_STEPS) TRIM_MIN_BYTES @@ -113,7 +113,7 @@ let rec getRandomPositionAux opt seed inputLen size = else getRandomPositionAux opt seed inputLen size let getRandomPosition opt seed size = - let inputLen = Seed.getCurInputLen seed + let inputLen = Seed.getCurLength seed getRandomPositionAux opt seed inputLen size let flipBit opt seed = @@ -191,7 +191,7 @@ let arithDword opt seed = Seed.updateBytesFrom seed pos newBytes let insertBytes seed = - let inputLen = Seed.getCurInputLen seed + let inputLen = Seed.getCurLength seed (* '+ 1' since we want to insert at the end of input, too. *) let insertPosition = random.Next(inputLen + 1) let insertSize = chooseBlockSize HAVOC_BLOCK_MEDIUM @@ -200,7 +200,7 @@ let insertBytes seed = Seed.insertBytesInto seed insertPosition insertContent let havocInsert seed = - let inputLen = Seed.getCurInputLen seed + let inputLen = Seed.getCurLength seed (* '+ 1' since we want to insert at the end of input, too. *) let insertPosition = random.Next(inputLen + 1) let insertContent = @@ -216,7 +216,7 @@ let havocInsert seed = Seed.insertBytesInto seed insertPosition insertContent let overwriteBytes seed = - let inputLen = Seed.getCurInputLen seed + let inputLen = Seed.getCurLength seed let writeSize = chooseBlockSize (inputLen / 2) // 'inputLen' in orig. havoc (* '+ 1' is needed to allow overwriting the last byte. *) let writePosition = random.Next(inputLen - writeSize + 1) @@ -232,13 +232,13 @@ let overwriteBytes seed = let removeBytes seed = // Assume input length is greater than 4. - let inputLen = Seed.getCurInputLen seed + let inputLen = Seed.getCurLength seed let removeSize = chooseBlockSize (inputLen / 2) // 'inputLen' in orig. havoc let removePosition = random.Next(inputLen - removeSize) Seed.removeBytesFrom seed removePosition removeSize let havocMutate opt seed = - let curInputLen = Seed.getCurInputLen seed + let curInputLen = Seed.getCurLength seed let maxN = if curInputLen < 2 then 8 elif curInputLen < 4 then 11 else 16 match random.Next(maxN) with | 0 -> flipBit opt seed @@ -263,8 +263,7 @@ let rec repRandomMutateAux seed opt depth depthLimit accumSeed = repRandomMutateAux seed opt (depth + 1) depthLimit accumSeed let repRandomMutate seed opt = - let seed = Seed.shuffleInputCursor seed - let curInputLen = Seed.getCurInputLen seed + let curInputLen = Seed.getCurLength seed let maxMutateN = max 1 (int (float curInputLen * MutateRatio)) let mutateN = min maxMutateN (1 <<< (1 + random.Next(HAVOC_STACK_POW))) let mutatedSeed = repRandomMutateAux seed opt 0 mutateN seed diff --git a/src/Fuzz/SeedQueue.fs b/src/Fuzz/SeedQueue.fs index 1656569..9065b2d 100644 --- a/src/Fuzz/SeedQueue.fs +++ b/src/Fuzz/SeedQueue.fs @@ -61,8 +61,7 @@ module RandFuzzQueue = let serializer = FsPickler.CreateBinarySerializer () let initialize queueDir = - let dummySeed = Seed.make Args [] 0 0 - { FavoredQueue = DurableQueue.initialize dummySeed + { FavoredQueue = DurableQueue.initialize Seed.dummy NormalQueue = FileQueue.initialize "rand-seed" queueDir LastMinimizedCount = 0 RemoveCount = 0 } diff --git a/src/Main/Decode.fs b/src/Main/Decode.fs deleted file mode 100644 index 299ea95..0000000 --- a/src/Main/Decode.fs +++ /dev/null @@ -1,67 +0,0 @@ -module Eclipser.Decode - -open System -open Argu -open Utils -open BytesUtils - -type Parser = ArgumentParser - -type DecoderCLI = - | [] [] [] InputDir of path: string - | [] [] [] OutputDir of path: string -with - interface IArgParserTemplate with - member s.Usage = - match s with - | InputDir _ -> "Directory of testcases to decode" - | OutputDir _ -> "Directory to store decoded outputs" - -type DecodeOption = { - TestcaseDir : string - OutputDir : string -} - -let parseDecodeOption args = - let cmdPrefix = "dotnet Eclipser.dll decode" - let parser = ArgumentParser.Create(programName = cmdPrefix) - let r = try parser.Parse(args) with - :? Argu.ArguParseException -> printLine (parser.PrintUsage()); exit 1 - { TestcaseDir = r.GetResult (<@ InputDir @>) - OutputDir = r.GetResult (<@ OutputDir @>) } - -/// Decode JSON-formatted/base64-encoded test case files and store decoded -/// payloads at output directory. -let run args = - let decodeOpt = parseDecodeOption args - let testcaseDir = decodeOpt.TestcaseDir - let outDir = decodeOpt.OutputDir - createDirectoryIfNotExists outDir - printLine ("Decoding test cases in : " + testcaseDir) - let decodedArgDir = outDir + "/decoded_args" - let decodedStdinDir = outDir + "/decoded_stdins" - let decodedFileContentDir = outDir + "/decoded_files" - let decodedFilePathDir = outDir + "/decoded_paths" - System.IO.Directory.CreateDirectory outDir |> ignore - System.IO.Directory.CreateDirectory decodedArgDir |> ignore - System.IO.Directory.CreateDirectory decodedStdinDir |> ignore - System.IO.Directory.CreateDirectory decodedFileContentDir |> ignore - System.IO.Directory.CreateDirectory decodedFilePathDir |> ignore - for tcFile in System.IO.Directory.EnumerateFiles(testcaseDir) do - let tokens = tcFile.Split([|'/'|], StringSplitOptions.RemoveEmptyEntries) - let tcName = tokens.[Array.length tokens - 1] - let tc = System.IO.File.ReadAllText tcFile |> TestCase.fromJSON - let argInput = - List.ofArray tc.Args - |> List.map (fun arg -> "\"" + escapeString arg + "\"") - |> String.concat " " - |> strToBytes - let stdInput = tc.StdIn - let fileInput = tc.FileContent - let filePath = strToBytes tc.FilePath - Executor.writeFile (sprintf "%s/%s" decodedArgDir tcName) argInput - Executor.writeFile (sprintf "%s/%s" decodedStdinDir tcName) stdInput - Executor.writeFile (sprintf "%s/%s" decodedFileContentDir tcName) fileInput - Executor.writeFile (sprintf "%s/%s" decodedFilePathDir tcName) filePath - - printLine "Decoding finished" diff --git a/src/Main/Main.fs b/src/Main/Main.fs deleted file mode 100644 index f2850a7..0000000 --- a/src/Main/Main.fs +++ /dev/null @@ -1,25 +0,0 @@ -module Eclipser.Main - -open Utils - -let printUsage () = - printLine "Usage: 'dotnet Eclipser.dll '" - printLine "fuzz : Mode for test case generation with fuzzing." - printLine " Use 'dotnet Eclipser.dll fuzz --help' for details." - printLine "replay : Mode for replaying generated test cases." - printLine " Use 'dotnet Eclipser.dll replay --help' for details." - printLine "decode : Mode for decoding generated test cases of JSON format." - printLine " Use 'dotnet Eclipser.dll decode --help' for details." - -let runMode (mode: string) optArgs = - match mode.ToLower() with - | "fuzz" -> Fuzz.run optArgs - | "replay" -> Replay.run optArgs - | "decode" -> Decode.run optArgs - | _ -> printUsage () - -[] -let main args = - if Array.length args <= 1 - then printUsage (); 1 - else runMode args.[0] args.[1..]; 0 diff --git a/src/Main/Replay.fs b/src/Main/Replay.fs deleted file mode 100644 index 523ee4d..0000000 --- a/src/Main/Replay.fs +++ /dev/null @@ -1,62 +0,0 @@ -module Eclipser.Replay - -open Argu -open Config -open Utils - -type ReplayerCLI = - | [] [] [] Program of path: string - | [] [] [] InputDir of path: string - // Options related to execution of program - | [] ExecTimeout of millisec: uint64 - | [] UsePty -with - interface IArgParserTemplate with - member s.Usage = - match s with - | Program _ -> "Target program for test case replaying" - | InputDir _ -> "Directory of testcases to replay" - // Options related to execution of program - | ExecTimeout _ -> "Timeout for each program execution (default:500)" - | UsePty _ -> "Use pseudo terminal for STDIN during the execution" - -type ReplayOption = { - Program : string - ExecTimeout : uint64 - UsePty : bool - TestcaseDir : string -} - -let parseReplayOption args = - let cmdPrefix = "dotnet Eclipser.dll replay" - let parser = ArgumentParser.Create (programName = cmdPrefix) - let r = try parser.Parse(args) with - :? Argu.ArguParseException -> printLine (parser.PrintUsage()); exit 1 - { Program = System.IO.Path.GetFullPath(r.GetResult (<@ Program @>)) - ExecTimeout = r.GetResult (<@ ExecTimeout @>, defaultValue = DefaultExecTO) - UsePty = r.Contains (<@ UsePty @>) - TestcaseDir = r.GetResult (<@ InputDir @>) } - -/// Replay test cases in the given directory on target program. This utility was -/// devised to estimate the code coverage of generated test cases, by replaying -/// test cases against a program compiled with 'gcov' options. -let run args = - let opt = parseReplayOption args - let program = opt.Program - let timeout = opt.ExecTimeout - let usePty = opt.UsePty - let testcaseDir = opt.TestcaseDir - assertFileExists program - Executor.initialize_exec Executor.TimeoutHandling.GDBQuit - printLine ("Start replaying test cases in : " + testcaseDir) - for file in System.IO.Directory.EnumerateFiles(testcaseDir) do - let tc = System.IO.File.ReadAllText file |> TestCase.fromJSON - printLine ("Replaying test case : " + file) - let args = Array.append [| program |] tc.Args - let argc = args.Length - let stdin = tc.StdIn - let stdinLen = stdin.Length - let files = Executor.setupFiles program tc true - Executor.exec(argc, args, stdinLen, stdin, timeout, usePty) |> ignore - removeFiles files - printLine "Replay finished" From ec45160c25fc701007cb613a25f2ca98ebee15b1 Mon Sep 17 00:00:00 2001 From: Jaeseung Choi Date: Sat, 19 Sep 2020 05:00:20 -0700 Subject: [PATCH 02/29] Update examples and testing scripts According to interface change of Eclipser. --- examples/getopt.c | 38 ---------------- examples/monoton.c | 10 +++-- examples/strcmp.c | 43 ------------------- examples/test_32bit_clang.sh | 10 +++++ examples/{test_32bit.sh => test_32bit_gcc.sh} | 4 +- examples/test_cmp_clang.sh | 4 +- examples/test_cmp_gcc.sh | 4 +- examples/test_fileonly.sh | 10 ----- examples/test_getopt.sh | 9 ---- examples/test_initseed.sh | 4 +- examples/test_loop.sh | 5 ++- examples/test_monoton_clang.sh | 4 +- examples/test_monoton_gcc.sh | 4 +- examples/test_motiv.sh | 7 ++- examples/test_nested.sh | 4 +- examples/test_strcmp_clang.sh | 9 ---- examples/test_strcmp_gcc.sh | 9 ---- examples/test_timeout.sh | 4 +- 18 files changed, 38 insertions(+), 144 deletions(-) delete mode 100644 examples/getopt.c delete mode 100644 examples/strcmp.c create mode 100755 examples/test_32bit_clang.sh rename examples/{test_32bit.sh => test_32bit_gcc.sh} (57%) delete mode 100755 examples/test_fileonly.sh delete mode 100755 examples/test_getopt.sh delete mode 100755 examples/test_strcmp_clang.sh delete mode 100755 examples/test_strcmp_gcc.sh diff --git a/examples/getopt.c b/examples/getopt.c deleted file mode 100644 index 5557617..0000000 --- a/examples/getopt.c +++ /dev/null @@ -1,38 +0,0 @@ -#include -#include -#include -#include - -int main(int argc, char ** argv){ - char c; - static struct option const long_options[] = - { - {"number", no_argument, NULL, 'n'}, - {"squeeze", no_argument, NULL, 's'}, - {"show", no_argument, NULL, 'A'}, - {NULL, 0, NULL, 0} - }; - - while ((c = getopt_long (argc, argv, "nsA", long_options, NULL)) != -1) - { - switch (c) - { - case 'n': - printf("Case 1\n"); // new path 2 - break; - - case 's': - printf("Case 2\n"); // new path 3 - break; - - case 'A': - printf("Case 3\n"); // new path 4 - break; - - default: - printf("failed, usage...\n"); - } - } - - return 0; -} diff --git a/examples/monoton.c b/examples/monoton.c index 1d5bb4c..ceeef03 100644 --- a/examples/monoton.c +++ b/examples/monoton.c @@ -1,6 +1,7 @@ /* Ab example with custom strcmp() function that requires binary search. */ #include +#include #include #include #include @@ -25,10 +26,8 @@ int main(int argc, char ** argv) { fd = open(argv[1], O_RDWR); read(fd, &i, sizeof(i)); - //if (i * i == 0Xfffc0004) // 0xfffe ^ 2 - // printf("Found new path 1\n"); if (i * i == 0x250b6984) // 0x6162 ^ 2 - printf("Found new path 1\n"); + printf("Found new path 1!\n"); read(fd, &j, sizeof(j)); j = ((j >> 24) & 0xff) | // move byte 3 to byte 0 @@ -41,7 +40,10 @@ int main(int argc, char ** argv) { n = read(fd, buf, 8); buf[n] = '\0'; if (my_strcmp(buf, "Good!") == 0) - printf("Found new path\n"); + printf("Found new path 3!\n"); + + if (strcmp(buf, "Bad!") == 0) + printf("Found new path 4!\n"); return 0; } diff --git a/examples/strcmp.c b/examples/strcmp.c deleted file mode 100644 index 6c733a4..0000000 --- a/examples/strcmp.c +++ /dev/null @@ -1,43 +0,0 @@ -/* A simple example to test whether Eclipser can handle strcmp() calls. */ - -#include -#include -#include -#include - -int main(int argc, char ** argv) { - char c; - short s; - int i; - int64_t i64; - - if (argc < 2) - return -1; - - if (strcmp(argv[1], "--option") == 0) - printf("Found new path 1!\n"); - - /* Note that this may fail in 32bit environment, since lowest 1-byte field of - * %esi, %dsi, %ebp, %esp are not accessible (%sil, %dil, %bpl, %spl are only - * accessible in 64bit environment, by using REX prefix). Therefore, our - * instrumentation code added in QEMU may not be able to generate 1-byte - * subtraction opcode, which results in the failure of flag-search on string. - */ - c = (char) strcmp(argv[1], "--char"); - if (c == 0) - printf("Found new path 2!\n"); - - s = (short) strcmp(argv[1], "--short"); - if (s == 0) - printf("Found new path 3!\n"); - - i = strcmp(argv[1], "--int32"); - if (i == 0) - printf("Found new path 4!\n"); - - i64 = strcmp(argv[1], "--int64"); - if (i64 == 0ll) - printf("Found new path 5!\n"); - - return 0; -} diff --git a/examples/test_32bit_clang.sh b/examples/test_32bit_clang.sh new file mode 100755 index 0000000..ad89884 --- /dev/null +++ b/examples/test_32bit_clang.sh @@ -0,0 +1,10 @@ +#!/bin/bash + +# Grey-box concolic should find more than 7 file input test cases that have node +# coverage gain. +clang cmp.c -o cmp.bin -static -g -m32 || exit 1 +rm -rf box +mkdir box +cd box +dotnet ../../build/Eclipser.dll \ + -p ../cmp.bin -t 5 -v 1 -o output -f input --arg input --architecture x86 diff --git a/examples/test_32bit.sh b/examples/test_32bit_gcc.sh similarity index 57% rename from examples/test_32bit.sh rename to examples/test_32bit_gcc.sh index c657991..08679c7 100755 --- a/examples/test_32bit.sh +++ b/examples/test_32bit_gcc.sh @@ -6,5 +6,5 @@ gcc cmp.c -o cmp.bin -static -g -m32 || exit 1 rm -rf box mkdir box cd box -dotnet ../../build/Eclipser.dll fuzz -p ../cmp.bin -t 20 -v 1 -o output \ - --src auto --maxarglen 8 --maxfilelen 16 --architecture x86 +dotnet ../../build/Eclipser.dll \ + -p ../cmp.bin -t 5 -v 1 -o output -f input --arg input --architecture x86 diff --git a/examples/test_cmp_clang.sh b/examples/test_cmp_clang.sh index dfe03ec..6fa17da 100755 --- a/examples/test_cmp_clang.sh +++ b/examples/test_cmp_clang.sh @@ -6,5 +6,5 @@ clang cmp.c -o cmp.bin -static -g || exit 1 rm -rf box mkdir box cd box -dotnet ../../build/Eclipser.dll fuzz -p ../cmp.bin -t 20 -v 1 -o output \ - --src auto --maxarglen 8 --maxfilelen 16 +dotnet ../../build/Eclipser.dll \ + -p ../cmp.bin -t 5 -v 1 -o output -f input --arg input diff --git a/examples/test_cmp_gcc.sh b/examples/test_cmp_gcc.sh index 109c536..951786c 100755 --- a/examples/test_cmp_gcc.sh +++ b/examples/test_cmp_gcc.sh @@ -6,5 +6,5 @@ gcc cmp.c -o cmp.bin -static -g || exit 1 rm -rf box mkdir box cd box -dotnet ../../build/Eclipser.dll fuzz -p ../cmp.bin -t 20 -v 1 -o output \ - --src auto --maxarglen 8 --maxfilelen 16 +dotnet ../../build/Eclipser.dll \ + -p ../cmp.bin -t 5 -v 1 -o output -f input --arg input diff --git a/examples/test_fileonly.sh b/examples/test_fileonly.sh deleted file mode 100755 index f6e5c99..0000000 --- a/examples/test_fileonly.sh +++ /dev/null @@ -1,10 +0,0 @@ -#!/bin/bash - -# Grey-box concolic should find more than 9 file input test cases that have node -# coverage gain. -gcc cmp.c -o cmp.bin -static -g || exit 1 # -static option for easier debugging -rm -rf box -mkdir box -cd box -dotnet ../../build/Eclipser.dll fuzz -p ../cmp.bin -t 5 -v 1 -o output \ - --src file --maxfilelen 16 -f input --initarg input diff --git a/examples/test_getopt.sh b/examples/test_getopt.sh deleted file mode 100755 index b592a20..0000000 --- a/examples/test_getopt.sh +++ /dev/null @@ -1,9 +0,0 @@ -#!/bin/bash - -# Grey-box concolic should find test cases "--number", "--squeeze", "--show". -gcc getopt.c -o getopt.bin -static -g || exit 1 -rm -rf box -mkdir box -cd box -dotnet ../../build/Eclipser.dll fuzz -p ../getopt.bin -t 15 -v 1 -o output \ - --src arg --maxarglen 10 diff --git a/examples/test_initseed.sh b/examples/test_initseed.sh index 068e61e..d1c3adb 100755 --- a/examples/test_initseed.sh +++ b/examples/test_initseed.sh @@ -8,5 +8,5 @@ mkdir box cd box mkdir seeds python -c 'print "B" * 16' > seeds/input -dotnet ../../build/Eclipser.dll fuzz -p ../cmp.bin -t 5 -v 1 -o output \ - --src file --maxfilelen 17 -i seeds -f input --initarg input +dotnet ../../build/Eclipser.dll \ + -p ../cmp.bin -t 5 -v 1 -i seeds -o output -f input --arg input diff --git a/examples/test_loop.sh b/examples/test_loop.sh index 11c5d4c..ae3009d 100755 --- a/examples/test_loop.sh +++ b/examples/test_loop.sh @@ -5,5 +5,6 @@ gcc loop.c -o loop.bin -static -g || exit 1 rm -rf box mkdir box cd box -dotnet ../../build/Eclipser.dll fuzz -p ../loop.bin -t 60 -v 1 -o output \ - --src auto --maxfilelen 16 --nsolve 10 +dotnet ../../build/Eclipser.dll \ + -p ../loop.bin -t 90 -v 1 -o output -f input --arg input --greyconcoliconly \ + --nsolve 10 diff --git a/examples/test_monoton_clang.sh b/examples/test_monoton_clang.sh index 8c45c29..ff66073 100755 --- a/examples/test_monoton_clang.sh +++ b/examples/test_monoton_clang.sh @@ -5,5 +5,5 @@ clang monoton.c -o monoton.bin -static -g || exit 1 rm -rf box mkdir box cd box -dotnet ../../build/Eclipser.dll fuzz -p ../monoton.bin -t 10 -v 1 -o output \ - --src auto --maxfilelen 16 +dotnet ../../build/Eclipser.dll \ + -p ../monoton.bin -t 10 -v 1 -o output -f input --arg input diff --git a/examples/test_monoton_gcc.sh b/examples/test_monoton_gcc.sh index fad9a15..a840dac 100755 --- a/examples/test_monoton_gcc.sh +++ b/examples/test_monoton_gcc.sh @@ -5,5 +5,5 @@ gcc monoton.c -o monoton.bin -static -g || exit 1 rm -rf box mkdir box cd box -dotnet ../../build/Eclipser.dll fuzz -p ../monoton.bin -t 10 -v 1 -o output \ - --src auto --maxfilelen 16 +dotnet ../../build/Eclipser.dll \ + -p ../monoton.bin -t 10 -v 1 -o output -f input --arg input diff --git a/examples/test_motiv.sh b/examples/test_motiv.sh index 0aaef25..ecb093d 100755 --- a/examples/test_motiv.sh +++ b/examples/test_motiv.sh @@ -1,10 +1,9 @@ -#!/bin/bash +!/bin/bash # Grey-box concolic should find a program-crashing test case within few seconds. gcc motiv.c -o motiv.bin -static -g || exit 1 rm -rf box mkdir box cd box -dotnet ../../build/Eclipser.dll fuzz -p ../motiv.bin -t 5 -v 1 -o output \ - --src file --maxfilelen 9 -f input - +dotnet ../../build/Eclipser.dll \ + -p ../motiv.bin -t 5 -v 1 -o output -f input --arg input diff --git a/examples/test_nested.sh b/examples/test_nested.sh index 9d9393f..eb9c6d3 100755 --- a/examples/test_nested.sh +++ b/examples/test_nested.sh @@ -6,5 +6,5 @@ gcc nested.c -o nested.bin -static -g || exit 1 rm -rf box mkdir box cd box -dotnet ../../build/Eclipser.dll fuzz -p ../nested.bin -t 20 -v 1 -o output \ - --src auto --maxfilelen 16 +dotnet ../../build/Eclipser.dll \ + -p ../nested.bin -t 20 -v 1 -o output -f input --arg input diff --git a/examples/test_strcmp_clang.sh b/examples/test_strcmp_clang.sh deleted file mode 100755 index d821583..0000000 --- a/examples/test_strcmp_clang.sh +++ /dev/null @@ -1,9 +0,0 @@ -#!/bin/bash - -# Grey-box concolic should find 6 test cases that have node coverage gain. -clang strcmp.c -o strcmp.bin -static -g || exit 1 -rm -rf box -mkdir box -cd box -dotnet ../../build/Eclipser.dll fuzz -p ../strcmp.bin -t 10 -v 1 -o output \ - --src arg --maxarglen 10 diff --git a/examples/test_strcmp_gcc.sh b/examples/test_strcmp_gcc.sh deleted file mode 100755 index d7359d2..0000000 --- a/examples/test_strcmp_gcc.sh +++ /dev/null @@ -1,9 +0,0 @@ -#!/bin/bash - -# Grey-box concolic should find 6 test cases that have node coverage gain -gcc strcmp.c -o strcmp.bin -static -g || exit 1 -rm -rf box -mkdir box -cd box -dotnet ../../build/Eclipser.dll fuzz -p ../strcmp.bin -t 10 -v 1 -o output \ - --src arg --maxarglen 10 diff --git a/examples/test_timeout.sh b/examples/test_timeout.sh index e25ac0a..f1f60ad 100755 --- a/examples/test_timeout.sh +++ b/examples/test_timeout.sh @@ -5,5 +5,5 @@ gcc timeout.c -o timeout.bin -static -g || exit 1 rm -rf box mkdir box cd box -dotnet ../../build/Eclipser.dll fuzz -p ../timeout.bin -t 5 -v 1 -o output \ - --src auto --maxfilelen 16 +dotnet ../../build/Eclipser.dll \ + -p ../timeout.bin -t 5 -v 1 -o output -f input --arg input From dc4317e15f9de4150d35e433315b90db32c22fb4 Mon Sep 17 00:00:00 2001 From: Jaeseung Choi Date: Mon, 21 Sep 2020 19:36:39 +0900 Subject: [PATCH 03/29] Remove random fuzzing module We will use AFL for random fuzzing. --- examples/test_loop.sh | 3 +- src/Core/Config.fs | 25 --- src/Core/Queue.fs | 137 ------------- src/Core/Seed.fs | 57 +----- src/Eclipser.fsproj | 3 - src/Executor/Executor.fs | 44 +---- src/Fuzz/Fuzz.fs | 151 +++++--------- src/Fuzz/GreyConcolic/GreyConcolic.fs | 20 -- src/Fuzz/Initialize.fs | 36 ---- src/Fuzz/Options.fs | 15 +- src/Fuzz/RandomFuzz.fs | 275 -------------------------- src/Fuzz/SeedQueue.fs | 136 ++----------- 12 files changed, 78 insertions(+), 824 deletions(-) delete mode 100644 src/Fuzz/Initialize.fs delete mode 100644 src/Fuzz/RandomFuzz.fs diff --git a/examples/test_loop.sh b/examples/test_loop.sh index ae3009d..79497ea 100755 --- a/examples/test_loop.sh +++ b/examples/test_loop.sh @@ -6,5 +6,4 @@ rm -rf box mkdir box cd box dotnet ../../build/Eclipser.dll \ - -p ../loop.bin -t 90 -v 1 -o output -f input --arg input --greyconcoliconly \ - --nsolve 10 + -p ../loop.bin -t 90 -v 1 -o output -f input --arg input --nsolve 10 diff --git a/src/Core/Config.fs b/src/Core/Config.fs index 5eb8a98..68ea766 100644 --- a/src/Core/Config.fs +++ b/src/Core/Config.fs @@ -3,28 +3,9 @@ module Eclipser.Config /// Default execution timeout of target program. let DefaultExecTO = 500UL -/// Trigger seed queue culling when the queue size increased by this ratio. -let SeedCullingThreshold = 1.2 - -/// Number of program execution allowed per each round of grey-box concolic -/// testing and random fuzz testing. -let ExecBudgetPerRound = 5000 - -/// Probability to use the queue of 'favored' seeds when selecting the next -/// seed to use for test case generation. -let FavoredSeedProb = 0.8 - -let MutateRatio = 0.2 -let RecentRoundN = 10 -let MinResrcRatio = 0.1 -let MaxResrcRatio = 1.0 - MinResrcRatio - /// Maximum length of chunk to try in grey-box concolic testing. let MaxChunkLen = 10 -/// In random fuzzing, try this number of random mutation for a chosen seed. -let RandSchTryCount = 100 - /// Call context sensitivity when measuring node coverage. let CtxSensitivity = 0 @@ -36,10 +17,4 @@ let MAX_INPUT_LEN = 1048576 let DurableQueueMaxSize = 1000 -let FileQueueMaxSize = 10000 - -let FileQueueMaxBytes = 2048UL * 1024UL * 1024UL // 2048 megabytes. - let BranchCombinationWindow = 6 - -let SkipFixedProb = 50 diff --git a/src/Core/Queue.fs b/src/Core/Queue.fs index 46d6442..88e8aaa 100644 --- a/src/Core/Queue.fs +++ b/src/Core/Queue.fs @@ -14,8 +14,6 @@ module Queue = let empty = { Enqueued = []; ToDequeue = [] } - let getSize q = q.Enqueued.Length + q.ToDequeue.Length - let isEmpty q = List.isEmpty q.Enqueued && List.isEmpty q.ToDequeue /// Enqueue an element to a queue. @@ -33,142 +31,7 @@ module Queue = | enq, deqHd :: deqTail -> (deqHd, { Enqueued = enq; ToDequeue = deqTail }) - /// Peek an element from a queue, without actually removing it from the queue. - let peek q = - match q.Enqueued, q.ToDequeue with - | [], [] -> raise EmptyException - | enq, [] -> List.head (List.rev enq) - | _, deqHd :: _ -> deqHd - - /// Drop the next element from a queue and return a shrunken queue. - let drop q = - match q.Enqueued, q.ToDequeue with - | [], [] -> raise EmptyException - | enq, [] -> { Enqueued = []; ToDequeue = List.tail (List.rev enq) } - | enq, _ :: deqTail -> { Enqueued = enq; ToDequeue = deqTail } - - let elements q = - q.ToDequeue @ (List.rev q.Enqueued) - let toString stringfy q = let elems = q.ToDequeue @ (List.rev q.Enqueued) let elemStrs = List.map stringfy elems String.concat "\n" elemStrs - -/// A durable queue, where elements are not volatile. Elements are fetched out -/// in a round-robin manner, but the elements are not removed from the queue -/// unless explicitly requested. -type DurableQueue<'a> = { - Elems : 'a array - Count : int - Finger : int // Next element to fetch. -} - -module DurableQueue = - - exception EmptyException - exception InvalidFingerException - - let initialize initElem = - { Elems = Array.create DurableQueueMaxSize initElem - Count = 0 - Finger = 0 } - - let getSize queue = queue.Count - - let isEmpty queue = - queue.Count = 0 - - let private isFull queue = - queue.Count >= queue.Elems.Length - - /// Enqueue an element to a queue. If the queue is already full, return the - /// same queue. - let enqueue queue elem = - if isFull queue then queue - else queue.Elems.[queue.Count] <- elem - { queue with Count = queue.Count + 1 } - - /// Remove an element from the queue. This functionality is useful when queue - /// should be managed in a compactly minimized state (e.g. seed culling). - let remove queue (idx, elem) = - if queue.Elems.[idx] <> elem then failwith "Element array messed up" - let shiftElem i = queue.Elems.[i] <- queue.Elems.[i + 1] - List.iter shiftElem (List.ofSeq { idx .. (queue.Count - 2) }) - let finger = queue.Finger - let newFinger = if idx < finger then finger - 1 else finger - let newCount = queue.Count - 1 - { queue with Finger = newFinger; Count = newCount } - - /// Fetch an element from the queue, while not removing the fetched element - /// from the queue. Raises DurableQueue.EmptyException if the queue is empty. - let fetch queue = - if queue.Count = 0 then raise EmptyException - if queue.Finger >= queue.Count then raise InvalidFingerException - let elem = queue.Elems.[queue.Finger] - let newFinger = (queue.Finger + 1) % queue.Count - let newQueue = { queue with Finger = newFinger } - (elem, newQueue) - -/// A file-system-based queue. Queue element is always a byte array (use Pickle -/// to encode a value into a byte array). -type FileQueue = { - Name : string - Directory : string - LowerIdx : int - UpperIdx : int - Finger : int - MaxCount : int -} - -module FileQueue = - - exception EmptyException - exception InvalidFingerException - - let initialize name directory = - ignore (System.IO.Directory.CreateDirectory(directory)) - { Name = name - Directory = directory - UpperIdx = 0 - LowerIdx = 0 - Finger = 0 - MaxCount = FileQueueMaxSize } - - let getSize queue = - queue.UpperIdx - queue.LowerIdx - - let private isFull queue = - queue.MaxCount <= getSize queue - - let isEmpty queue = - getSize queue = 0 - - let private isValidFinger queue = - queue.LowerIdx <= queue.Finger && queue.Finger < queue.UpperIdx - - let add queue elemBytes = - let filePath = sprintf "%s/%s-%d" queue.Directory queue.Name queue.UpperIdx - System.IO.File.WriteAllBytes(filePath, elemBytes) - let newUpperIdx = queue.UpperIdx + 1 - { queue with UpperIdx = newUpperIdx } - - /// Enqueue an element to a queue. If the queue is already full, return the - /// same queue. - let enqueue queue elemBytes = - if isFull queue then queue else add queue elemBytes - - /// Dequeue an element from a queue. Raises FileQueue.EmptyException if the - /// queue is empty. - let dequeue queue = - if isEmpty queue then raise EmptyException - if not (isValidFinger queue) then raise InvalidFingerException - let lowerIdx = queue.LowerIdx - let finger = queue.Finger - let filePath = sprintf "%s/%s-%d" queue.Directory queue.Name lowerIdx - let elemBytes = System.IO.File.ReadAllBytes(filePath) - System.IO.File.Delete filePath - let newLowerIdx = lowerIdx + 1 - let newFinger = max newLowerIdx finger - let newQueue = { queue with Finger = newFinger; LowerIdx = newLowerIdx } - (elemBytes, newQueue) \ No newline at end of file diff --git a/src/Core/Seed.fs b/src/Core/Seed.fs index 5eacbda..44a8af9 100644 --- a/src/Core/Seed.fs +++ b/src/Core/Seed.fs @@ -101,14 +101,14 @@ module Seed = | Some idx -> byteVals'.Length - idx - 1 // Auxiliary function for queryUpdateBound() - let private queryUpdateBoundRight (byteVals: ByteVal []) byteCursor maxLen = + let private queryUpdateBoundRight (byteVals: ByteVal []) byteCursor = let byteVals' = if byteCursor + MaxChunkLen < byteVals.Length then byteVals.[byteCursor .. byteCursor + MaxChunkLen] else byteVals.[byteCursor .. ] // We use an heuristic to bound update until the adjacent *fixed* ByteVal. match Array.tryFindIndex ByteVal.isFixed byteVals' with - | None -> min MaxChunkLen (maxLen - byteCursor) + | None -> MaxChunkLen | Some idx -> idx /// Find the maximum length that can be updated for grey-box concolic testing. @@ -118,7 +118,7 @@ module Seed = match direction with | Stay -> failwith "queryUpdateBound() cannot be called with 'Stay'" | Left -> queryUpdateBoundLeft byteVals byteCursor - | Right -> queryUpdateBoundRight byteVals byteCursor MAX_INPUT_LEN + | Right -> queryUpdateBoundRight byteVals byteCursor /// Get adjacent concrete byte values, toward the given direction. let queryNeighborBytes seed direction = @@ -171,53 +171,6 @@ module Seed = newByteVals.[byteCursor] <- byteVal { seed with ByteVals = newByteVals } - /// Update the ByteVal at given offset of the seed. - let updateByteValAt seed pos byteVal = - let curByteVals = seed.ByteVals - let newByteVals = Array.copy curByteVals - newByteVals.[pos] <- byteVal - { seed with ByteVals = newByteVals } - - /// Update the bytes of the seed, starting from the given offset. - /// Approximate path conditions of the updated ByteVals are abandoned. - let updateBytesFrom seed pos bytes = - let curByteVals = seed.ByteVals - let newByteVals = Array.copy curByteVals - Array.iteri (fun i b -> newByteVals.[pos + i] <- Undecided b) bytes - { seed with ByteVals = newByteVals } - - /// Flip the bit at the given byte/bit offset of the seed. - let flipBitAt seed bytePos bitPos = - let curByteVals = seed.ByteVals - let newByteVals = Array.copy curByteVals - let curByteVal = curByteVals.[bytePos] - let curByte = ByteVal.getConcreteByte curByteVal - let newByte = curByte ^^^ ((byte 1) <<< bitPos) - newByteVals.[bytePos] <- Undecided newByte - { seed with ByteVals = newByteVals } - - /// Insert bytes at the given offset of the seed. - let insertBytesInto seed pos bytes = - let curByteVals = seed.ByteVals - let curBytesLen = Array.length curByteVals - let headByteVals = curByteVals.[0 .. (pos - 1)] - let tailByteVals = curByteVals.[pos .. (curBytesLen - 1)] - let byteVals = Array.map (fun b -> Undecided b) bytes - let newByteVals = Array.concat [headByteVals; byteVals; tailByteVals] - let newByteVals = if Array.length newByteVals > MAX_INPUT_LEN - then newByteVals.[.. (MAX_INPUT_LEN - 1)] - else newByteVals - { seed with ByteVals = newByteVals } - - /// Remove bytes starting from the given offset of the seed. - let removeBytesFrom seed pos n = - let curByteVals = seed.ByteVals - let curBytesLen = Array.length curByteVals - let headByteVals = curByteVals.[0 .. (pos - 1)] - let tailByteVals = curByteVals.[(pos + n) .. (curBytesLen - 1)] - let newByteVals = Array.append headByteVals tailByteVals - { seed with ByteVals = newByteVals } - (************************* Cursor update functions *************************) /// Set the byte cursor position of the seed. @@ -282,11 +235,11 @@ module Seed = /// Return seeds with byte cursor updated to point unfixed ByteVal. Probing /// should occur in both left side and right side. - let relocateCursor isFromConcolic seed = + let relocateCursor seed = let curByteVal = getCurByteVal seed let leftwardSeed = setByteCursorDir seed Left let leftwardSeeds = // Avoid sampling at the same offset. - if isFromConcolic && ByteVal.isSampledByte curByteVal then + if ByteVal.isSampledByte curByteVal then match stepCursor leftwardSeed with | None -> [] | Some s -> [ s ] else [ leftwardSeed ] diff --git a/src/Eclipser.fsproj b/src/Eclipser.fsproj index 10958e1..99177d2 100644 --- a/src/Eclipser.fsproj +++ b/src/Eclipser.fsproj @@ -28,8 +28,6 @@ - - @@ -37,7 +35,6 @@ - diff --git a/src/Executor/Executor.fs b/src/Executor/Executor.fs index 5be5d54..04eadda 100644 --- a/src/Executor/Executor.fs +++ b/src/Executor/Executor.fs @@ -10,7 +10,7 @@ open Options let private WHITES = [| ' '; '\t'; '\n' |] /// Kinds of QEMU instrumentor. Each instrumentor serves different purposes. -type Tracer = Coverage | BranchAt | BranchAll | BBCount +type Tracer = Coverage | Branch | BBCount /// Specifies the method to handle execution timeout. It is necessary to use GDB /// when replaying test cases against a gcov-compiled binary, to log coverage @@ -57,10 +57,8 @@ let selectTracer tracer arch = match tracer, arch with | Coverage, X86 -> coverageTracerX86 | Coverage, X64 -> coverageTracerX64 - | BranchAt, X86 -> branchTracerX86 - | BranchAt, X64 -> branchTracerX64 - | BranchAll, X86 -> branchTracerX86 - | BranchAll, X64 -> branchTracerX64 + | Branch, X86 -> branchTracerX86 + | Branch, X64 -> branchTracerX64 | BBCount, X86 -> bbCountTracerX86 | BBCount, X64 -> bbCountTracerX64 @@ -102,7 +100,7 @@ let initialize opt = let pidCoverage = init_forkserver_coverage(args.Length, args, opt.ExecTimeout) if pidCoverage = -1 then failwith "Failed to initialize fork server for coverage tracer" - let branchTracer = selectTracer BranchAll opt.Architecture + let branchTracer = selectTracer Branch opt.Architecture let args = Array.append [|branchTracer; opt.TargetProg|] cmdLine let pidBranch = init_forkserver_branch (args.Length, args, opt.ExecTimeout) if pidBranch = -1 then @@ -116,27 +114,6 @@ let cleanup () = removeFile coverageLog removeFile dbgLog -(*** Statistics ***) - -let mutable totalExecutions = 0 -let mutable phaseExecutions = 0 -let getTotalExecutions () = totalExecutions -let getPhaseExecutions () = phaseExecutions -let resetPhaseExecutions () = phaseExecutions <- 0 - -(*** Resource scheduling ***) - -let mutable allowedResource = 0 -let allocateResource n = allowedResource <- n -let isResourceExhausted () = allowedResource <= 0 -let incrExecutionCount tracerTyp = - // TODO: Add calibration mode like AFL, and estimate the weight of each tracer - // type based on execution time. - let diminish = if tracerTyp = BranchAll then 4 else 1 - allowedResource <- allowedResource - diminish - totalExecutions <- totalExecutions + 1 - phaseExecutions <- phaseExecutions + 1 - let abandonForkServer () = log "Abandon fork server" forkServerEnabled <- false @@ -242,7 +219,6 @@ let private readBranchTrace opt filename tryVal = (*** Tracer execute functions ***) let private runTracer tracerType opt (stdin: byte array) = - incrExecutionCount tracerType let targetProg = opt.TargetProg let timeout = opt.ExecTimeout let usePty = opt.UsePty @@ -253,14 +229,12 @@ let private runTracer tracerType opt (stdin: byte array) = exec(argc, args, stdin.Length, stdin, timeout, usePty) let private runCoverageTracerForked opt (stdin: byte array) = - incrExecutionCount Coverage let timeout = opt.ExecTimeout let signal = exec_fork_coverage(timeout, stdin.Length, stdin) if signal = Signal.ERROR then abandonForkServer () signal -let private runBranchTracerForked tracerType opt (stdin: byte array) = - incrExecutionCount tracerType +let private runBranchTracerForked opt (stdin: byte array) = let timeout = opt.ExecTimeout let signal = exec_fork_branch(timeout, stdin.Length, stdin) if signal = Signal.ERROR then abandonForkServer () @@ -306,8 +280,8 @@ let getBranchTrace opt seed tryVal = set_env("CK_FEED_IDX", "0") setupFile seed let stdin = prepareStdIn seed - if forkServerEnabled then runBranchTracerForked BranchAll opt stdin - else runTracer BranchAll opt stdin + if forkServerEnabled then runBranchTracerForked opt stdin + else runTracer Branch opt stdin |> ignore let pathHash = parseExecHash pathHashLog let branchTrace = readBranchTrace opt branchTraceLog tryVal @@ -321,8 +295,8 @@ let getBranchInfoAt opt seed tryVal targPoint = set_env("CK_FEED_IDX", sprintf "%016x" targPoint.Idx) setupFile seed let stdin = prepareStdIn seed - if forkServerEnabled then runBranchTracerForked BranchAt opt stdin - else runTracer BranchAt opt stdin + if forkServerEnabled then runBranchTracerForked opt stdin + else runTracer Branch opt stdin |> ignore let pathHash = parseExecHash pathHashLog let branchInfoOpt = diff --git a/src/Fuzz/Fuzz.fs b/src/Fuzz/Fuzz.fs index 643d0f3..4ad1564 100644 --- a/src/Fuzz/Fuzz.fs +++ b/src/Fuzz/Fuzz.fs @@ -1,118 +1,60 @@ module Eclipser.Fuzz -open Config +open System.Threading open Utils open Options -let rec makeItemsAux opt isFromConcolic accConcolic accRandom items = - match items with - | [] -> (accConcolic, accRandom) - | (priority, seed) :: tailItems -> - let concItems = Seed.relocateCursor isFromConcolic seed - |> List.map (fun s -> (priority, s)) - let randItems = [(priority, seed)] - let accConcolic = concItems @ accConcolic - let accRandom = randItems @ accRandom - makeItemsAux opt isFromConcolic accConcolic accRandom tailItems +let private initializeSeeds opt = + if opt.InputDir = "" then [Seed.make opt.FuzzSource] + else System.IO.Directory.EnumerateFiles opt.InputDir // Obtain file list + |> List.ofSeq // Convert list to array + |> List.map System.IO.File.ReadAllBytes // Read in file contents + |> List.map (Seed.makeWith opt.FuzzSource) // Create seed with content -let makeItems opt isFromConcolic seeds = - let concItems, randItems = makeItemsAux opt isFromConcolic [] [] seeds - (List.rev concItems, List.rev randItems) +let private preprocessAux opt seed = + let newEdgeN, pathHash, edgeHash, exitSig = Executor.getCoverage opt seed + let isNewPath = Manager.save opt seed newEdgeN pathHash edgeHash exitSig true + if newEdgeN > 0 then Some (Favored, seed) + elif isNewPath then Some (Normal, seed) + else None -/// Allocate testing resource for each strategy (grey-box concolic testing and -/// random fuzz testing). Resource is managed through 'the number of allowed -/// program execution'. If the number of instrumented program execution exceeds -/// the specified number, the strategy will be switched. -let allocResource opt = - if opt.GreyConcolicOnly then (ExecBudgetPerRound, 0) - elif opt.RandFuzzOnly then (0, ExecBudgetPerRound) - else - let concolicEff = GreyConcolic.evaluateEfficiency () - let randFuzzEff = RandomFuzz.evaluateEfficiency () - let concolicRatio = concolicEff / (concolicEff + randFuzzEff) - // Bound alloc ratio with 'MinResourceAlloc', to avoid extreme biasing - let concolicRatio = max MinResrcRatio (min MaxResrcRatio concolicRatio) - let randFuzzRatio = 1.0 - concolicRatio - let totalBudget = ExecBudgetPerRound - let greyConcBudget = int (float totalBudget * concolicRatio) - let randFuzzBudget = int (float totalBudget * randFuzzRatio) - (greyConcBudget, randFuzzBudget) +let private preprocess opt seeds = + log "[*] Total %d initial seeds" (List.length seeds) + let items = List.choose (preprocessAux opt) seeds + let favoredCount = List.filter (fst >> (=) Favored) items |> List.length + let normalCount = List.filter (fst >> (=) Normal) items |> List.length + log "[*] %d initial items with high priority" favoredCount + log "[*] %d initial items with low priority" normalCount + items + +let private makeNewItems seeds = + let expand (pr, seed) = List.map (fun s -> (pr, s)) (Seed.relocateCursor seed) + List.collect expand seeds + +let private makeSteppedItems pr seed = + match Seed.proceedCursor seed with + | None -> [] + | Some s -> [(pr, s)] -let rec greyConcolicLoop opt concQ randQ = - if Executor.isResourceExhausted () || ConcolicQueue.isEmpty concQ - then (concQ, randQ) +let rec private fuzzLoop opt seedQueue = + if SeedQueue.isEmpty seedQueue then + Thread.Sleep(5000) + fuzzLoop opt seedQueue else - let pr, seed, concQ = ConcolicQueue.dequeue concQ + let pr, seed, seedQueue = SeedQueue.dequeue seedQueue if opt.Verbosity >= 1 then log "Grey-box concolic on %A seed : %s" pr (Seed.toString seed) let newSeeds = GreyConcolic.run seed opt // Move cursors of newly generated seeds. - let newItemsForConc, newItemsForRand = makeItems opt true newSeeds + let newItems = makeNewItems newSeeds // Also generate seeds by just stepping the cursor of original seed. - let steppedItems = - match Seed.proceedCursor seed with - | None -> [] - | Some s -> [(pr, s)] - let concQ = List.fold ConcolicQueue.enqueue concQ newItemsForConc - let concQ = List.fold ConcolicQueue.enqueue concQ steppedItems - // Note that 'Stepped' seeds are not enqueued for random fuzzing. - let randQ = List.fold RandFuzzQueue.enqueue randQ newItemsForRand - greyConcolicLoop opt concQ randQ - -let repeatGreyConcolic opt concQ randQ concolicBudget = - if opt.Verbosity >= 1 then log "Grey-box concoclic testing phase starts" - Executor.allocateResource concolicBudget - Executor.resetPhaseExecutions () - let pathNumBefore = Manager.getPathCount () - let concQ, randQ = greyConcolicLoop opt concQ randQ - let pathNumAfter = Manager.getPathCount () - let concolicExecNum = Executor.getPhaseExecutions () - let concolicNewPathNum = pathNumAfter - pathNumBefore - GreyConcolic.updateStatus opt concolicExecNum concolicNewPathNum - (concQ, randQ) - -let rec randFuzzLoop opt concQ randQ = - // Random fuzzing seeds are involatile, so don't have to check emptiness. - if Executor.isResourceExhausted () - then (concQ, randQ) - else - let pr, seed, randQ = RandFuzzQueue.dequeue randQ - if opt.Verbosity >= 1 then - log "Random fuzzing on %A seed : %s" pr (Seed.toString seed) - let newSeeds = RandomFuzz.run seed opt - // Move cursors of newly generated seeds. - let newItemsForConc, newItemsForRand = makeItems opt false newSeeds - let concQ = List.fold ConcolicQueue.enqueue concQ newItemsForConc - let randQ = List.fold RandFuzzQueue.enqueue randQ newItemsForRand - randFuzzLoop opt concQ randQ - -let repeatRandFuzz opt concQ randQ randFuzzBudget = - if opt.Verbosity >= 1 then log "Random fuzzing phase starts" - Executor.allocateResource randFuzzBudget - Executor.resetPhaseExecutions () - let pathNumBefore = Manager.getPathCount () - let concQ, randQ = randFuzzLoop opt concQ randQ - let pathNumAfter = Manager.getPathCount () - let randExecNum = Executor.getPhaseExecutions () - let randNewPathNum = pathNumAfter - pathNumBefore - RandomFuzz.updateStatus opt randExecNum randNewPathNum - (concQ, randQ) - -let rec fuzzLoop opt concQ randQ = - // Note that random fuzzing queue is always non-empty, since it's involatile. - if not (opt.GreyConcolicOnly && ConcolicQueue.isEmpty concQ) then - let concolicBudget, randFuzzBudget = allocResource opt - // Perform grey-box concolic testing - let concQ, randQ = repeatGreyConcolic opt concQ randQ concolicBudget + let steppedItems = makeSteppedItems pr seed + let seedQueue = List.fold SeedQueue.enqueue seedQueue newItems + let seedQueue = List.fold SeedQueue.enqueue seedQueue steppedItems // Perform random fuzzing - let concQ, randQ = repeatRandFuzz opt concQ randQ randFuzzBudget - // Minimize random-fuzzing queue if # of seeds increased considerably - let randQ = if RandFuzzQueue.timeToMinimize randQ - then RandFuzzQueue.minimize randQ opt - else randQ - fuzzLoop opt concQ randQ + fuzzLoop opt seedQueue -let fuzzingTimer timeoutSec queueDir = async { +let private fuzzingTimer timeoutSec = async { let timespan = System.TimeSpan(0, 0, 0, timeoutSec) System.Threading.Thread.Sleep(timespan ) printLine "Fuzzing timeout expired." @@ -120,7 +62,6 @@ let fuzzingTimer timeoutSec queueDir = async { Manager.printStatistics () log "Done, clean up and exit..." Executor.cleanup () - removeDir queueDir exit (0) } @@ -134,9 +75,11 @@ let main args = createDirectoryIfNotExists opt.OutDir Manager.initialize opt.OutDir Executor.initialize opt - let queueDir = sprintf "%s/.internal" opt.OutDir - let greyConcQueue, randFuzzQueue = Initialize.initQueue opt queueDir + let emptyQueue = SeedQueue.initialize () + let initialSeeds = initializeSeeds opt + let initItems = preprocess opt initialSeeds + let initQueue = List.fold SeedQueue.enqueue emptyQueue initItems log "[*] Fuzzing starts" - Async.Start (fuzzingTimer opt.Timelimit queueDir) - fuzzLoop opt greyConcQueue randFuzzQueue + Async.Start (fuzzingTimer opt.Timelimit) + fuzzLoop opt initQueue 0 // Unreachable diff --git a/src/Fuzz/GreyConcolic/GreyConcolic.fs b/src/Fuzz/GreyConcolic/GreyConcolic.fs index 738f4c2..37a2aff 100644 --- a/src/Fuzz/GreyConcolic/GreyConcolic.fs +++ b/src/Fuzz/GreyConcolic/GreyConcolic.fs @@ -1,28 +1,8 @@ module Eclipser.GreyConcolic -open Config open Utils open Options -// Mutable variables for statistics management. -let mutable private recentExecNums: Queue = Queue.empty -let mutable private recentNewPathNums: Queue = Queue.empty - -let updateStatus opt execN newPathN = - let recentExecNums' = if Queue.getSize recentExecNums > RecentRoundN - then Queue.drop recentExecNums - else recentExecNums - recentExecNums <- Queue.enqueue recentExecNums' execN - let recentNewPathNums' = if Queue.getSize recentNewPathNums > RecentRoundN - then Queue.drop recentNewPathNums - else recentNewPathNums - recentNewPathNums <- Queue.enqueue recentNewPathNums' newPathN - -let evaluateEfficiency () = - let execNum = List.sum (Queue.elements recentExecNums) - let newPathNum = List.sum (Queue.elements recentNewPathNums) - if execNum = 0 then 1.0 else float newPathNum / float execNum - let printFoundSeed verbosity seed newEdgeN = let edgeStr = if newEdgeN > 0 then sprintf "(%d new edges) " newEdgeN else "" if verbosity >= 1 then diff --git a/src/Fuzz/Initialize.fs b/src/Fuzz/Initialize.fs deleted file mode 100644 index 155d364..0000000 --- a/src/Fuzz/Initialize.fs +++ /dev/null @@ -1,36 +0,0 @@ -module Eclipser.Initialize - -open Utils -open Options - -let private initializeSeeds opt = - if opt.InputDir = "" then [Seed.make opt.FuzzSource] - else System.IO.Directory.EnumerateFiles opt.InputDir // Obtain file list - |> List.ofSeq // Convert list to array - |> List.map System.IO.File.ReadAllBytes // Read in file contents - |> List.map (Seed.makeWith opt.FuzzSource) // Create seed with content - -let private preprocessAux opt seed = - let newEdgeN, pathHash, edgeHash, exitSig = Executor.getCoverage opt seed - let isNewPath = Manager.save opt seed newEdgeN pathHash edgeHash exitSig true - if newEdgeN > 0 then Some (Favored, seed) - elif isNewPath then Some (Normal, seed) - else None - -let private preprocess opt seeds = - log "[*] Total %d initial seeds" (List.length seeds) - let items = List.choose (preprocessAux opt) seeds - let favoredCount = List.filter (fst >> (=) Favored) items |> List.length - let normalCount = List.filter (fst >> (=) Normal) items |> List.length - log "[*] %d initial items with high priority" favoredCount - log "[*] %d initial items with low priority" normalCount - items - -let initQueue opt queueDir = - let initialSeeds = initializeSeeds opt - let initItems = preprocess opt initialSeeds - let greyConcQueue = ConcolicQueue.initialize queueDir - let greyConcQueue = List.fold ConcolicQueue.enqueue greyConcQueue initItems - let randFuzzQueue = RandFuzzQueue.initialize queueDir - let randFuzzQueue = List.fold RandFuzzQueue.enqueue randFuzzQueue initItems - (greyConcQueue, randFuzzQueue) diff --git a/src/Fuzz/Options.fs b/src/Fuzz/Options.fs index 82988a9..6642d79 100644 --- a/src/Fuzz/Options.fs +++ b/src/Fuzz/Options.fs @@ -20,8 +20,6 @@ type FuzzerCLI = // Options related to fuzzing process | [] NSolve of int | [] NSpawn of int - | [] GreyConcolicOnly - | [] RandFuzzOnly with interface IArgParserTemplate with member s.Usage = @@ -30,8 +28,8 @@ with | Verbose _ -> "Verbosity level to control debug messages (default:0)." | Timelimit _ -> "Timeout for fuzz testing (in seconds)." | OutputDir _ -> "Directory to store testcase outputs." - // Options related to seed initialization | InputDir _ -> "Directory containing initial seeds." + // Options related to seed initialization | Arg _ -> "Command-line argument of program under test." | Filepath _ -> "File input's (fixed) path" // Options related to execution of program @@ -44,8 +42,6 @@ with "the paper." | NSpawn _ -> "Number of byte values to initially spawn in grey-box " + "concolic testing. 'N_spawn' parameter in the paper." - | GreyConcolicOnly -> "Perform grey-box concolic testing only." - | RandFuzzOnly -> "Perform random fuzzing only." type FuzzOption = { Verbosity : int @@ -63,8 +59,6 @@ type FuzzOption = { // Options related to test case generation NSolve : int NSpawn : int - GreyConcolicOnly : bool - RandFuzzOnly : bool } let parseFuzzOption (args: string array) = @@ -88,13 +82,8 @@ let parseFuzzOption (args: string array) = Arg = r.GetResult (<@ Arg @>, defaultValue = "") // Options related to test case generation NSolve = r.GetResult(<@ NSolve @>, defaultValue = 600) - NSpawn = r.GetResult(<@ NSpawn @>, defaultValue = 10) - GreyConcolicOnly = r.Contains(<@ GreyConcolicOnly @>) - RandFuzzOnly = r.Contains(<@ RandFuzzOnly @>) } + NSpawn = r.GetResult(<@ NSpawn @>, defaultValue = 10) } let validateFuzzOption opt = - if opt.GreyConcolicOnly && opt.RandFuzzOnly then - printLine "Cannot specify '--greyconcoliconly' and 'randfuzzonly' together." - exit 1 if opt.NSpawn < 3 then failwith "Should provide N_spawn greater than or equal to 3" diff --git a/src/Fuzz/RandomFuzz.fs b/src/Fuzz/RandomFuzz.fs deleted file mode 100644 index f7d61b8..0000000 --- a/src/Fuzz/RandomFuzz.fs +++ /dev/null @@ -1,275 +0,0 @@ -module Eclipser.RandomFuzz - -open Config -open Utils -open BytesUtils -open Options - -// Constants -let TRIM_MIN_BYTES = 4 -let TRIM_START_STEPS = 16 -let TRIM_END_STEPS = 1024 - -let ARITH_MAX = 35 -let BLOCK_SIZE_MAX = 32 - -let HAVOC_STACK_POW = 7 -let HAVOC_BLOCK_SMALL = 32 -let HAVOC_BLOCK_MEDIUM = 128 -let HAVOC_BLOCK_LARGE = 1500 -let HAVOC_BLOCK_XLARGE = 32768 - -// Mutable variables for statistics management. -let mutable private recentExecNums: Queue = Queue.empty -let mutable private recentNewPathNums: Queue = Queue.empty - -let updateStatus opt execN newPathN = - let recentExecNums' = if Queue.getSize recentExecNums > RecentRoundN - then Queue.drop recentExecNums - else recentExecNums - recentExecNums <- Queue.enqueue recentExecNums' execN - let recentNewPathNums' = if Queue.getSize recentNewPathNums > RecentRoundN - then Queue.drop recentNewPathNums - else recentNewPathNums - recentNewPathNums <- Queue.enqueue recentNewPathNums' newPathN - -let evaluateEfficiency () = - let execNum = List.sum (Queue.elements recentExecNums) - let newPathNum = List.sum (Queue.elements recentNewPathNums) - if execNum = 0 then 1.0 else float newPathNum / float execNum - -let chooseBlockSize limit = - // XXX: Original havoc mutation in AFL also depends on queue cycle - let minVal, maxVal = - match random.Next(4) with - | 0 -> 1, HAVOC_BLOCK_SMALL - | 1 | 2 -> HAVOC_BLOCK_SMALL, HAVOC_BLOCK_MEDIUM // P=1/3 in original havoc - | 3 -> if random.Next(10) <> 0 - then HAVOC_BLOCK_MEDIUM, HAVOC_BLOCK_LARGE - else HAVOC_BLOCK_LARGE, HAVOC_BLOCK_XLARGE - | _ -> failwith "Unexpected random value" - let minVal = if minVal >= limit then 1 else minVal - let maxVal = min maxVal limit - minVal + random.Next(maxVal - minVal + 1) - -let rec roundUpExpAux accVal input = - if accVal >= input then accVal else roundUpExpAux (accVal <<< 1) input - -let roundUpExp input = - roundUpExpAux 1 input - -let rec trimAux opt accSeed edgeHash trimMinSize trimSize pos = - if trimSize < trimMinSize then - accSeed // Cannot lower trimSize anymore, time to stop - elif pos + trimSize >= Seed.getCurLength accSeed then - (* Reached end, retry with more fine granularity (reset idx to 0). *) - trimAux opt accSeed edgeHash trimMinSize (trimSize / 2) 0 - else let trySeed = Seed.removeBytesFrom accSeed pos trimSize - let tryEdgeHash = Executor.getEdgeHash opt trySeed - if tryEdgeHash = edgeHash then // Trimming succeeded. - (* Caution : Next trimming position is not 'pos + trimSize' *) - trimAux opt trySeed edgeHash trimMinSize trimSize pos - else (* Trimming failed, move on to next position *) - let newPos = (pos + trimSize) - trimAux opt accSeed edgeHash trimMinSize trimSize newPos - -let trim opt edgeHash seed = - let inputLen = Seed.getCurLength seed - let inputLenRounded = roundUpExp inputLen - let trimSize = max (inputLenRounded / TRIM_START_STEPS) TRIM_MIN_BYTES - let trimMinSize = max (inputLenRounded / TRIM_END_STEPS) TRIM_MIN_BYTES - let trimmedSeed = trimAux opt seed edgeHash trimMinSize trimSize 0 - // Should adjust byte cursor again within a valid range. - Seed.shuffleByteCursor trimmedSeed - -let printFoundSeed verbosity seed newEdgeN = - let edgeStr = if newEdgeN > 0 then sprintf "(%d new edges) " newEdgeN else "" - if verbosity >= 1 then - log "[*] Found by random fuzzing %s: %s" edgeStr (Seed.toString seed) - elif verbosity >= 0 then - log "[*] Found by random fuzzing %s" edgeStr - -let evalSeedsAux opt accItems seed = - let newEdgeN, pathHash, edgeHash, exitSig = Executor.getCoverage opt seed - let isNewPath = Manager.save opt seed newEdgeN pathHash edgeHash exitSig false - if newEdgeN > 0 then printFoundSeed opt.Verbosity seed newEdgeN - if isNewPath && not (Signal.isTimeout exitSig) && not (Signal.isCrash exitSig) - then - let priority = if newEdgeN > 0 then Favored else Normal - // Trimming is only for seeds that found new edges - let seed' = if newEdgeN > 0 then trim opt edgeHash seed else seed - (priority, seed') :: accItems - else accItems - -let evalSeeds opt seeds = - List.fold (evalSeedsAux opt) [] seeds - -(* Functions related to mutation *) - -let rec getRandomPositionAux opt seed inputLen size = - let bytePos = random.Next(inputLen + 1 - size) - if Seed.isUnfixedByteAt seed bytePos then bytePos - elif random.Next(100) >= SkipFixedProb then bytePos // XXX - else getRandomPositionAux opt seed inputLen size - -let getRandomPosition opt seed size = - let inputLen = Seed.getCurLength seed - getRandomPositionAux opt seed inputLen size - -let flipBit opt seed = - let bytePos = getRandomPosition opt seed 1 - let bitPos = random.Next(8) - Seed.flipBitAt seed bytePos bitPos - -let randomByte opt seed = - let pos = getRandomPosition opt seed 1 - let newByte = allBytes.[random.Next(allBytes.Length)] - let newByteVal = Undecided newByte - Seed.updateByteValAt seed pos newByteVal - -let i_byte = [|0x80uy; 0xffuy; 0uy; 1uy; 16uy; 32uy; 64uy; 100uy; 127uy|] - -let interestingByte opt seed = - let pos = getRandomPosition opt seed 1 - let newBytes = [| i_byte.[random.Next(Array.length i_byte)] |] - Seed.updateBytesFrom seed pos newBytes - -let i_word = [|0x80us; 0xffus; 0us; 1us; 16us; 32us; 64us; 100us; 127us; - 0x8000us; 0xff7fus; 128us; 255us; 256us; 512us; 1000us; - 1024us; 4096us; 32767us|] - -let interestingWord opt seed = - // Assume input length is greater than 2. - let pos = getRandomPosition opt seed 2 - let endian = if random.Next(2) = 0 then LE else BE - let interest = i_word.[random.Next(Array.length i_word)] - let newBytes = uIntToBytes endian 2 (uint32 interest) - Seed.updateBytesFrom seed pos newBytes - -let i_dword = [|0x80u; 0xffu; 0u; 1u; 16u; 32u; 64u; 100u; 127u; 0x8000u; - 0xff7fu; 128u; 255u; 256u; 512u; 1000u; 1024u; 4096u; 32767u; - 0x80000000u; 0xfa0000fau; 0xffff7fffu; 0x8000u; 0xffffu; - 0x10000u; 0x5ffff05u; 0x7fffffffu|] - -let interestingDWord opt seed = - // Assume input length is greater than 4. - let pos = getRandomPosition opt seed 4 - let endian = if random.Next(2) = 0 then LE else BE - let interest = i_dword.[random.Next(Array.length i_dword)] - let newBytes = uIntToBytes endian 4 interest - Seed.updateBytesFrom seed pos newBytes - -let arithByte opt seed = - let pos = getRandomPosition opt seed 1 - let curVal = uint32 (Seed.getConcreteByteAt seed pos) - let delta = uint32 (1 + random.Next(ARITH_MAX)) - let newVal = if random.Next(2) = 0 then curVal + delta else curVal - delta - let newByte = byte (newVal &&& 0xffu) - let newByteVal = Undecided newByte - Seed.updateByteValAt seed pos newByteVal - -let arithWord opt seed = - // Assume input length is greater than 2. - let pos = getRandomPosition opt seed 2 - let endian = if random.Next(2) = 0 then LE else BE - let curBytes = Seed.getConcreteBytesFrom seed pos 2 - let curVal = bytesToUInt endian curBytes - let delta = uint32 (1 + random.Next(ARITH_MAX)) - let newVal = if random.Next(2) = 0 then curVal + delta else curVal - delta - let newBytes = uIntToBytes endian 2 newVal - Seed.updateBytesFrom seed pos newBytes - -let arithDword opt seed = - // Assume input length is greater than 4. - let pos = getRandomPosition opt seed 4 - let endian = if random.Next(2) = 0 then LE else BE - let curBytes = Seed.getConcreteBytesFrom seed pos 4 - let curVal = bytesToUInt endian curBytes - let delta = uint32 (1 + random.Next(ARITH_MAX)) - let newVal = if random.Next(2) = 0 then curVal + delta else curVal - delta - let newBytes = uIntToBytes endian 4 newVal - Seed.updateBytesFrom seed pos newBytes - -let insertBytes seed = - let inputLen = Seed.getCurLength seed - (* '+ 1' since we want to insert at the end of input, too. *) - let insertPosition = random.Next(inputLen + 1) - let insertSize = chooseBlockSize HAVOC_BLOCK_MEDIUM - let genNewByte () = allBytes.[random.Next(allBytes.Length)] - let insertContent = Array.init insertSize (fun _ -> genNewByte()) - Seed.insertBytesInto seed insertPosition insertContent - -let havocInsert seed = - let inputLen = Seed.getCurLength seed - (* '+ 1' since we want to insert at the end of input, too. *) - let insertPosition = random.Next(inputLen + 1) - let insertContent = - if random.Next(2) = 0 then // 1/4 in original havoc - let insertSize = chooseBlockSize HAVOC_BLOCK_XLARGE - let newByte = allBytes.[random.Next(allBytes.Length)] - Array.init insertSize (fun _ -> newByte) - else - let insertSize = chooseBlockSize inputLen - (* '+ 1' is needed to allow copying from the last byte. *) - let copyFrom = random.Next(inputLen - insertSize + 1) - Seed.getConcreteBytesFrom seed copyFrom insertSize - Seed.insertBytesInto seed insertPosition insertContent - -let overwriteBytes seed = - let inputLen = Seed.getCurLength seed - let writeSize = chooseBlockSize (inputLen / 2) // 'inputLen' in orig. havoc - (* '+ 1' is needed to allow overwriting the last byte. *) - let writePosition = random.Next(inputLen - writeSize + 1) - let writeContent = - if random.Next(2) <> 0 then // 1/4 in original havoc - let newByte = allBytes.[random.Next(allBytes.Length)] - Array.init writeSize (fun _ -> newByte) - else - (* '+ 1' is needed to allow copying from the last byte. *) - let copyFrom = random.Next(inputLen - writeSize + 1) - Seed.getConcreteBytesFrom seed copyFrom writeSize - Seed.updateBytesFrom seed writePosition writeContent - -let removeBytes seed = - // Assume input length is greater than 4. - let inputLen = Seed.getCurLength seed - let removeSize = chooseBlockSize (inputLen / 2) // 'inputLen' in orig. havoc - let removePosition = random.Next(inputLen - removeSize) - Seed.removeBytesFrom seed removePosition removeSize - -let havocMutate opt seed = - let curInputLen = Seed.getCurLength seed - let maxN = if curInputLen < 2 then 8 elif curInputLen < 4 then 11 else 16 - match random.Next(maxN) with - | 0 -> flipBit opt seed - | 1 -> randomByte opt seed - | 2 -> havocInsert seed - | 3 -> insertBytes seed - | 4 -> overwriteBytes seed - | 5 | 6 -> arithByte opt seed - | 7 -> interestingByte opt seed - // Require input length >= 2 - | 8 | 9 -> arithWord opt seed - | 10 -> interestingWord opt seed - // Require input length >= 4 - | 11 | 12 -> arithDword opt seed - | 13 -> interestingDWord opt seed - | 14 | 15 -> removeBytes seed - | _ -> failwith "Wrong mutation code" - -let rec repRandomMutateAux seed opt depth depthLimit accumSeed = - if depth >= depthLimit then accumSeed else - let accumSeed = havocMutate opt accumSeed - repRandomMutateAux seed opt (depth + 1) depthLimit accumSeed - -let repRandomMutate seed opt = - let curInputLen = Seed.getCurLength seed - let maxMutateN = max 1 (int (float curInputLen * MutateRatio)) - let mutateN = min maxMutateN (1 <<< (1 + random.Next(HAVOC_STACK_POW))) - let mutatedSeed = repRandomMutateAux seed opt 0 mutateN seed - Seed.shuffleByteCursor mutatedSeed - -let run seed opt = - let tryCnt = Config.RandSchTryCount - let mutatedSeeds = List.init tryCnt (fun _ -> repRandomMutate seed opt) - evalSeeds opt mutatedSeeds diff --git a/src/Fuzz/SeedQueue.fs b/src/Fuzz/SeedQueue.fs index 9065b2d..defe88b 100644 --- a/src/Fuzz/SeedQueue.fs +++ b/src/Fuzz/SeedQueue.fs @@ -1,140 +1,32 @@ namespace Eclipser -open MBrace.FsPickler -open Config -open Utils -open Options - type ConcolicQueue = { - FavoredQueue : Queue - NormalQueue : FileQueue + Favoreds : Queue + Normals : Queue } -module ConcolicQueue = - - let serializer = FsPickler.CreateBinarySerializer () +module SeedQueue = - let initialize queueDir = - { FavoredQueue = Queue.empty - NormalQueue = FileQueue.initialize "concolic-seed" queueDir } + let initialize () = + { Favoreds = Queue.empty + Normals = Queue.empty } let isEmpty queue = - Queue.isEmpty queue.FavoredQueue && FileQueue.isEmpty queue.NormalQueue + Queue.isEmpty queue.Favoreds && Queue.isEmpty queue.Normals let enqueue queue (priority, seed) = match priority with - | Favored -> - let newFavoredQueue = Queue.enqueue queue.FavoredQueue seed - { queue with FavoredQueue = newFavoredQueue } - | Normal -> - let seedBytes = serializer.Pickle seed - let newNormalQueue = FileQueue.enqueue queue.NormalQueue seedBytes - { queue with NormalQueue = newNormalQueue } + | Favored -> { queue with Favoreds = Queue.enqueue queue.Favoreds seed } + | Normal -> { queue with Normals = Queue.enqueue queue.Normals seed } let dequeue queue = - let favoredQueue = queue.FavoredQueue - let normalQueue = queue.NormalQueue - let queueSelection = - if FileQueue.isEmpty normalQueue then Favored - elif Queue.isEmpty favoredQueue then Normal - else Favored - match queueSelection with - | Favored -> - let seed, newFavoredQueue = Queue.dequeue favoredQueue - let newQueue = { queue with FavoredQueue = newFavoredQueue } - (Favored, seed, newQueue) - | Normal -> - let seedBytes, newNormalQueue = FileQueue.dequeue normalQueue - let seed = serializer.UnPickle seedBytes - let newQueue = { queue with NormalQueue = newNormalQueue } - (Normal, seed, newQueue) - -type RandFuzzQueue = { - FavoredQueue : DurableQueue - NormalQueue : FileQueue - LastMinimizedCount : int - RemoveCount : int -} - -module RandFuzzQueue = - - let serializer = FsPickler.CreateBinarySerializer () - - let initialize queueDir = - { FavoredQueue = DurableQueue.initialize Seed.dummy - NormalQueue = FileQueue.initialize "rand-seed" queueDir - LastMinimizedCount = 0 - RemoveCount = 0 } - - let enqueue queue (priority, seed) = + let priority = if Queue.isEmpty queue.Favoreds then Normal else Favored match priority with | Favored -> - let newFavoredQueue = DurableQueue.enqueue queue.FavoredQueue seed - { queue with FavoredQueue = newFavoredQueue } - | Normal -> - let seedBytes = serializer.Pickle seed - let newNormalQueue = FileQueue.enqueue queue.NormalQueue seedBytes - { queue with NormalQueue = newNormalQueue } - - let dequeue queue = - let favoredQueue = queue.FavoredQueue - let normalQueue = queue.NormalQueue - let queueSelection = - if FileQueue.isEmpty normalQueue then Favored - elif random.NextDouble() < FavoredSeedProb then Favored - else Normal - match queueSelection with - | Favored -> - let seed, newFavoredQueue = DurableQueue.fetch favoredQueue - let newQueue = { queue with FavoredQueue = newFavoredQueue } + let seed, newFavoreds = Queue.dequeue queue.Favoreds + let newQueue = { queue with Favoreds = newFavoreds } (Favored, seed, newQueue) | Normal -> - let seedBytes, newNormalQueue = FileQueue.dequeue normalQueue - let seed = serializer.UnPickle seedBytes - let newQueue = { queue with NormalQueue = newNormalQueue } + let seed, newNormals = Queue.dequeue queue.Normals + let newQueue = { queue with Normals = newNormals } (Normal, seed, newQueue) - - let timeToMinimize queue = - let curSize = queue.FavoredQueue.Count - let prevSize = queue.LastMinimizedCount - curSize > int (float prevSize * SeedCullingThreshold) - - let rec findRedundantsGreedyAux queue seedEntries accRedundantSeeds = - if List.isEmpty seedEntries then accRedundantSeeds - else - (* Choose an entry that has largest number of covered edges *) - let getEdgeCount (idx, seed, edges) = Set.count edges - let seedEntriesSorted = List.sortByDescending getEdgeCount seedEntries - let _, _, chosenEdges = List.head seedEntriesSorted - let seedEntries = List.tail seedEntriesSorted - (* Now update (i.e. subtract edge set) seed entries *) - let subtractEdges edges (i, s, ns) = (i, s, Set.difference ns edges) - let seedEntries = List.map (subtractEdges chosenEdges) seedEntries - (* If the edge set entry is empty, it means that seed is redundant *) - let redundantEntries, seedEntries = - List.partition (fun (i, s, ns) -> Set.isEmpty ns) seedEntries - let redundantSeeds = List.map (fun (i, s, _) -> (i, s)) redundantEntries - let accRedundantSeeds = redundantSeeds @ accRedundantSeeds - findRedundantsGreedyAux queue seedEntries accRedundantSeeds - - let findRedundantsGreedy seeds queue opt = - let seedEntries = - List.fold - (fun accSets (idx, seed) -> - (idx, seed, Executor.getEdgeSet opt seed) :: accSets - ) [] seeds - findRedundantsGreedyAux queue seedEntries [] - - let minimize queue opt = - let favoredQueue = queue.FavoredQueue - let seeds = favoredQueue.Elems.[0 .. favoredQueue.Count - 1] - |> Array.mapi (fun i seed -> (i, seed)) - |> List.ofArray - let seedsToRemove = findRedundantsGreedy seeds favoredQueue opt - // Note that we should remove larger index first - let newFavoredQueue = List.sortByDescending fst seedsToRemove - |> List.fold DurableQueue.remove favoredQueue - { queue with FavoredQueue = newFavoredQueue - LastMinimizedCount = newFavoredQueue.Count - RemoveCount = queue.RemoveCount + seedsToRemove.Length } - From 290759415ea97279044326f33bdc255a742becbd Mon Sep 17 00:00:00 2001 From: Jaeseung Choi Date: Tue, 22 Sep 2020 11:28:29 +0900 Subject: [PATCH 04/29] Refactor and cleanup code - Reorganize directory, file and module structures. - Remove obsolete code. --- Makefile | 2 +- src/{Fuzz/GreyConcolic => Core}/BranchInfo.fs | 20 ++--- src/{Executor => Core}/Executor.fs | 22 ------ src/Core/Input.fs | 7 -- src/{Fuzz => Core}/Options.fs | 0 src/Core/Seed.fs | 3 + src/Core/Typedef.fs | 52 ++++++++++--- src/{Executor => Core}/libexec.c | 0 src/Eclipser.fsproj | 27 +++---- src/Executor/Signal.fs | 31 -------- src/Executor/Syscall.fs | 76 ------------------- src/Fuzz/Fuzz.fs | 34 +++++++-- src/Fuzz/GreyConcolic/GreyConcolic.fs | 43 ----------- src/Fuzz/Manager.fs | 15 ++-- src/Fuzz/SeedQueue.fs | 37 ++++++++- src/{Fuzz => }/GreyConcolic/BranchTrace.fs | 4 +- src/{Fuzz => }/GreyConcolic/BranchTree.fs | 0 src/GreyConcolic/GreyConcolic.fs | 18 +++++ src/{Fuzz => }/GreyConcolic/LinearEquation.fs | 0 .../GreyConcolic/LinearInequality.fs | 0 src/{Fuzz => }/GreyConcolic/Linearity.fs | 0 src/{Fuzz => }/GreyConcolic/Monotonicity.fs | 0 src/{Fuzz => }/GreyConcolic/PathConstraint.fs | 0 src/{Fuzz => }/GreyConcolic/Solve.fs | 0 24 files changed, 156 insertions(+), 235 deletions(-) rename src/{Fuzz/GreyConcolic => Core}/BranchInfo.fs (100%) rename src/{Executor => Core}/Executor.fs (94%) delete mode 100644 src/Core/Input.fs rename src/{Fuzz => Core}/Options.fs (100%) rename src/{Executor => Core}/libexec.c (100%) delete mode 100644 src/Executor/Signal.fs delete mode 100644 src/Executor/Syscall.fs delete mode 100644 src/Fuzz/GreyConcolic/GreyConcolic.fs rename src/{Fuzz => }/GreyConcolic/BranchTrace.fs (95%) rename src/{Fuzz => }/GreyConcolic/BranchTree.fs (100%) create mode 100644 src/GreyConcolic/GreyConcolic.fs rename src/{Fuzz => }/GreyConcolic/LinearEquation.fs (100%) rename src/{Fuzz => }/GreyConcolic/LinearInequality.fs (100%) rename src/{Fuzz => }/GreyConcolic/Linearity.fs (100%) rename src/{Fuzz => }/GreyConcolic/Monotonicity.fs (100%) rename src/{Fuzz => }/GreyConcolic/PathConstraint.fs (100%) rename src/{Fuzz => }/GreyConcolic/Solve.fs (100%) diff --git a/Makefile b/Makefile index 81358c8..d50e49a 100644 --- a/Makefile +++ b/Makefile @@ -41,7 +41,7 @@ $(QEMUDIR)/.compiled_x64: $(SHDIR)/.compiled $(QEMUDIR)/.prepared cd $(QEMUDIR) && ./build_qemu_x64.sh @touch $@ -$(BUILDDIR)/libexec.dll: src/Executor/libexec.c +$(BUILDDIR)/libexec.dll: src/Core/libexec.c gcc -O3 -shared -fPIC $< -o $@ Eclipser: $(BUILDDIR)/libexec.dll diff --git a/src/Fuzz/GreyConcolic/BranchInfo.fs b/src/Core/BranchInfo.fs similarity index 100% rename from src/Fuzz/GreyConcolic/BranchInfo.fs rename to src/Core/BranchInfo.fs index 8b7e0d2..d5dae53 100644 --- a/src/Fuzz/GreyConcolic/BranchInfo.fs +++ b/src/Core/BranchInfo.fs @@ -4,16 +4,6 @@ open Utils type CompareType = Equality | SignedSize | UnsignedSize -type BranchInfo = { - InstAddr : uint64 - BrType : CompareType - TryVal : bigint - OpSize : int - Oprnd1 : uint64 - Oprnd2 : uint64 - Distance : bigint -} - type BranchPoint = { Addr : uint64 Idx : int @@ -24,6 +14,16 @@ type Context = { ByteDir : Direction } +type BranchInfo = { + InstAddr : uint64 + BrType : CompareType + TryVal : bigint + OpSize : int + Oprnd1 : uint64 + Oprnd2 : uint64 + Distance : bigint +} + module BranchInfo = let toString diff --git a/src/Executor/Executor.fs b/src/Core/Executor.fs similarity index 94% rename from src/Executor/Executor.fs rename to src/Core/Executor.fs index 04eadda..c833e40 100644 --- a/src/Executor/Executor.fs +++ b/src/Core/Executor.fs @@ -242,17 +242,6 @@ let private runBranchTracerForked opt (stdin: byte array) = (*** Top-level tracer executor functions ***) -let getEdgeHash opt seed = - set_env("CK_MODE", string (int CoverageTracerMode.EdgeHash)) - setupFile seed - let stdin = prepareStdIn seed - if forkServerEnabled then runCoverageTracerForked opt stdin - else runTracer Coverage opt stdin - |> ignore - let edgeHash = parseExecHash coverageLog - clearFile seed - edgeHash - let getCoverage opt seed = set_env("CK_MODE", string (int CoverageTracerMode.CountNewEdge)) setupFile seed @@ -264,17 +253,6 @@ let getCoverage opt seed = clearFile seed (newEdgeCnt, pathHash, edgeHash, exitSig) -let getEdgeSet opt seed = - set_env("CK_MODE", string (int CoverageTracerMode.EdgeSet)) - setupFile seed - let stdin = prepareStdIn seed - if forkServerEnabled then runCoverageTracerForked opt stdin - else runTracer Coverage opt stdin - |> ignore - let edgeSet = readEdgeSet opt coverageLog - clearFile seed - edgeSet - let getBranchTrace opt seed tryVal = set_env("CK_FEED_ADDR", "0") set_env("CK_FEED_IDX", "0") diff --git a/src/Core/Input.fs b/src/Core/Input.fs deleted file mode 100644 index 31699cf..0000000 --- a/src/Core/Input.fs +++ /dev/null @@ -1,7 +0,0 @@ -namespace Eclipser - -open System -open Config -open Utils -open BytesUtils - diff --git a/src/Fuzz/Options.fs b/src/Core/Options.fs similarity index 100% rename from src/Fuzz/Options.fs rename to src/Core/Options.fs diff --git a/src/Core/Seed.fs b/src/Core/Seed.fs index 44a8af9..c9510f4 100644 --- a/src/Core/Seed.fs +++ b/src/Core/Seed.fs @@ -3,6 +3,9 @@ namespace Eclipser open Config open Utils +/// Direction that the cursor of a 'Seed' should move toward. +type Direction = Stay | Left | Right + /// An input that consists of an array of ByteVals. type Seed = { /// An array of ByteVal elements. diff --git a/src/Core/Typedef.fs b/src/Core/Typedef.fs index a37619c..91948e0 100644 --- a/src/Core/Typedef.fs +++ b/src/Core/Typedef.fs @@ -6,6 +6,18 @@ type Signedness = Signed | Unsigned type Sign = Positive | Negative | Zero +/// Specifies which input source to fuzz. +type InputSource = + /// Fuzz standard input source. + | StdInput + /// Fuzz file input source. + | FileInput of filepath: string + +/// Priority of found seed. A seed that increased edge coverage is assigned +/// 'Favored' priority, while a seed that increased path coverage is assigned +/// 'Normal' priority. +type Priority = Favored | Normal + /// Architecture of target program to fuzz. type Arch = X86 | X64 @@ -19,17 +31,33 @@ module Arch = | "x64" -> X64 | _ -> raise UnsupportedArchException -/// Specifies which input source to fuzz. -type InputSource = - /// Fuzz standard input source. - | StdInput - /// Fuzz file input source. - | FileInput of filepath: string +/// Signal returned by program execution. +type Signal = + | ERROR = -1 + | NORMAL = 0 + | SIGILL = 4 + | SIGABRT = 6 + | SIGFPE = 8 + | SIGSEGV = 11 + | SIGALRM = 14 -/// Direction that the cursor of a 'Seed' should move toward. -type Direction = Stay | Left | Right +module Signal = + let isCrash signal = + match signal with + | Signal.SIGSEGV | Signal.SIGILL | Signal.SIGABRT-> true + | _ -> false -/// Priority of found seed. A seed that increased edge coverage is assigned -/// 'Favored' priority, while a seed that increased path coverage is assigned -/// 'Normal' priority. -type Priority = Favored | Normal \ No newline at end of file + let isSegfault signal = + signal = Signal.SIGSEGV + + let isIllegal signal = + signal = Signal.SIGILL + + let isFPE signal = + signal = Signal.SIGFPE + + let isAbort signal = + signal = Signal.SIGABRT + + let isTimeout signal = + signal = Signal.SIGALRM diff --git a/src/Executor/libexec.c b/src/Core/libexec.c similarity index 100% rename from src/Executor/libexec.c rename to src/Core/libexec.c diff --git a/src/Eclipser.fsproj b/src/Eclipser.fsproj index 99177d2..cd13486 100644 --- a/src/Eclipser.fsproj +++ b/src/Eclipser.fsproj @@ -9,24 +9,21 @@ - + - - - - - + + - - - - - - - - - + + + + + + + + + diff --git a/src/Executor/Signal.fs b/src/Executor/Signal.fs deleted file mode 100644 index 873cea1..0000000 --- a/src/Executor/Signal.fs +++ /dev/null @@ -1,31 +0,0 @@ -namespace Eclipser - -type Signal = - | ERROR = -1 - | NORMAL = 0 - | SIGILL = 4 - | SIGABRT = 6 - | SIGFPE = 8 - | SIGSEGV = 11 - | SIGALRM = 14 - -module Signal = - let isCrash signal = - match signal with - | Signal.SIGSEGV | Signal.SIGILL | Signal.SIGABRT-> true - | _ -> false - - let isSegfault signal = - signal = Signal.SIGSEGV - - let isIllegal signal = - signal = Signal.SIGILL - - let isFPE signal = - signal = Signal.SIGFPE - - let isAbort signal = - signal = Signal.SIGABRT - - let isTimeout signal = - signal = Signal.SIGALRM diff --git a/src/Executor/Syscall.fs b/src/Executor/Syscall.fs deleted file mode 100644 index d5df3ef..0000000 --- a/src/Executor/Syscall.fs +++ /dev/null @@ -1,76 +0,0 @@ -namespace Eclipser - -open System -open System.IO -open System.Collections.Generic -open Utils -open Options - -type Syscall = - | Open of (string * int) - | Dup of (int * int) - | Read of int - -module Syscall = - let rec checkFileReadAux (filename:string) fdMap syscalls = - match syscalls with - | [] -> false - | Open (fname, fd) :: tail_syscalls -> - let fdMap = Map.add fd fname fdMap - checkFileReadAux filename fdMap tail_syscalls - | Dup (old_fd, new_fd) :: tail_syscalls -> - let fdMap = - try Map.add new_fd (Map.find old_fd fdMap) fdMap with - | :? KeyNotFoundException -> fdMap - checkFileReadAux filename fdMap tail_syscalls - | Read fd :: tail_syscalls -> - (* Check if name of the file just read is equal to the 'filename' arg. - * If they unmatch, continue with recursion. Recall that newline character - * was escaped in 'fname', not to break syscall log format. - *) - let fname = try Map.find fd fdMap with | :? KeyNotFoundException -> "" - let filenameEscaped = escapeWhiteSpace filename - System.IO.Path.GetFileName fname = filenameEscaped - && not (System.IO.Directory.Exists fname) - || checkFileReadAux filename fdMap tail_syscalls - - let checkFileRead syscalls filename = - let fdMap = - Map.empty - |> Map.add 0 ".STDIN" - |> Map.add 1 ".STDOUT" - |> Map.add 2 ".STDERR" - checkFileReadAux filename fdMap syscalls - - let checkStdInputRead syscalls = - checkFileRead syscalls ".STDIN" - - let findFileInput syscalls args = - try List.findIndex (checkFileRead syscalls) (List.ofArray args) with - | :? KeyNotFoundException -> -1 - - let parseSyscallLog targProg (content:string) = - try - match List.ofSeq (content.Split ' ') with - | ["read"; fdStr] -> Some (Read (int fdStr)) - | ["dup"; oldFdStr; newFdStr] -> Some (Dup (int oldFdStr, int newFdStr)) - | ["open"; fdStr; filename] -> - let filepath = try Path.GetFullPath filename with _ -> "" - if filepath <> "" && filepath <> targProg then - Some (Open (filename, int fdStr)) // Caution : not 'filepath' here. - else None - | _ -> (log "[Warning] Wrong syscall log file format : %s" content; None) - with :? FormatException -> - (log "[Warning] Wrong syscall log file format : %s" content; None) - - let checkInputSource targProg args syscallLog = - let syscalls = List.choose (parseSyscallLog targProg) syscallLog - let hasStdInput = checkStdInputRead syscalls - let fileArgIdx = findFileInput syscalls args - let inputSrcs = - if hasStdInput then Set.singleton InputSrc.StdInput else Set.empty - let inputSrcs = - if fileArgIdx <> -1 then - Set.add (InputSrc.FileInput fileArgIdx) inputSrcs - else inputSrcs - inputSrcs diff --git a/src/Fuzz/Fuzz.fs b/src/Fuzz/Fuzz.fs index 4ad1564..4f24ff4 100644 --- a/src/Fuzz/Fuzz.fs +++ b/src/Fuzz/Fuzz.fs @@ -4,6 +4,23 @@ open System.Threading open Utils open Options +let private printFoundSeed verbosity seed newEdgeN = + let edgeStr = if newEdgeN > 0 then sprintf "(%d new edges) " newEdgeN else "" + if verbosity >= 1 then + log "[*] Found by grey-box concolic %s: %s" edgeStr (Seed.toString seed) + elif verbosity >= 0 then + log "[*] Found by grey-box concolic %s" edgeStr + +let private evalSeed opt seed = + let newEdgeN, pathHash, edgeHash, exitSig = Executor.getCoverage opt seed + let isNewPath = Manager.save opt seed newEdgeN pathHash edgeHash exitSig false + if newEdgeN > 0 then printFoundSeed opt.Verbosity seed newEdgeN + let isAbnormal = Signal.isTimeout exitSig || Signal.isCrash exitSig + if isNewPath && not isAbnormal then + let priority = if newEdgeN > 0 then Favored else Normal + Some priority + else None + let private initializeSeeds opt = if opt.InputDir = "" then [Seed.make opt.FuzzSource] else System.IO.Directory.EnumerateFiles opt.InputDir // Obtain file list @@ -27,9 +44,12 @@ let private preprocess opt seeds = log "[*] %d initial items with low priority" normalCount items -let private makeNewItems seeds = - let expand (pr, seed) = List.map (fun s -> (pr, s)) (Seed.relocateCursor seed) - List.collect expand seeds +let private makeNewItems opt seeds = + let collector seed = + match evalSeed opt seed with + | None -> [] + | Some pr -> List.map (fun s -> (pr, s)) (Seed.relocateCursor seed) + List.collect collector seeds let private makeSteppedItems pr seed = match Seed.proceedCursor seed with @@ -41,14 +61,14 @@ let rec private fuzzLoop opt seedQueue = Thread.Sleep(5000) fuzzLoop opt seedQueue else - let pr, seed, seedQueue = SeedQueue.dequeue seedQueue + let priority, seed, seedQueue = SeedQueue.dequeue seedQueue if opt.Verbosity >= 1 then - log "Grey-box concolic on %A seed : %s" pr (Seed.toString seed) + log "Grey-box concolic on %A seed : %s" priority (Seed.toString seed) let newSeeds = GreyConcolic.run seed opt // Move cursors of newly generated seeds. - let newItems = makeNewItems newSeeds + let newItems = makeNewItems opt newSeeds // Also generate seeds by just stepping the cursor of original seed. - let steppedItems = makeSteppedItems pr seed + let steppedItems = makeSteppedItems priority seed let seedQueue = List.fold SeedQueue.enqueue seedQueue newItems let seedQueue = List.fold SeedQueue.enqueue seedQueue steppedItems // Perform random fuzzing diff --git a/src/Fuzz/GreyConcolic/GreyConcolic.fs b/src/Fuzz/GreyConcolic/GreyConcolic.fs deleted file mode 100644 index 37a2aff..0000000 --- a/src/Fuzz/GreyConcolic/GreyConcolic.fs +++ /dev/null @@ -1,43 +0,0 @@ -module Eclipser.GreyConcolic - -open Utils -open Options - -let printFoundSeed verbosity seed newEdgeN = - let edgeStr = if newEdgeN > 0 then sprintf "(%d new edges) " newEdgeN else "" - if verbosity >= 1 then - log "[*] Found by grey-box concolic %s: %s" edgeStr (Seed.toString seed) - elif verbosity >= 0 then - log "[*] Found by grey-box concolic %s" edgeStr - -let evalSeedsAux opt accSeeds seed = - let newEdgeN, pathHash, edgeHash, exitSig = Executor.getCoverage opt seed - let isNewPath = Manager.save opt seed newEdgeN pathHash edgeHash exitSig false - if newEdgeN > 0 then printFoundSeed opt.Verbosity seed newEdgeN - if isNewPath && not (Signal.isTimeout exitSig) && not (Signal.isCrash exitSig) - then let priority = if newEdgeN > 0 then Favored else Normal - (priority, seed) :: accSeeds - else accSeeds - -let evalSeeds opt items = - List.fold (evalSeedsAux opt) [] items |> List.rev // To preserve order - -let checkByProducts opt spawnedSeeds = - evalSeeds opt spawnedSeeds - -let run seed opt = - let curByteVal = Seed.getCurByteVal seed - let minByte, maxByte = ByteVal.getMinMax curByteVal seed.Source - if minByte = maxByte then - let seedStr = Seed.toString seed - failwithf "Cursor pointing to Fixed ByteVal %s" seedStr - let minVal, maxVal = bigint (int minByte), bigint (int maxByte) - let branchTraces, spawnSeeds = BranchTrace.collect seed opt minVal maxVal - let byteDir = Seed.getByteCursorDir seed - let bytes = Seed.queryNeighborBytes seed byteDir - let ctx = { Bytes = bytes; ByteDir = byteDir } - let branchTree = BranchTree.make opt ctx branchTraces - let branchTree = BranchTree.selectAndRepair opt branchTree - GreySolver.clearSolutionCache () - let solutions = GreySolver.solve seed opt byteDir branchTree - evalSeeds opt solutions @ checkByProducts opt spawnSeeds diff --git a/src/Fuzz/Manager.fs b/src/Fuzz/Manager.fs index 8889906..40161cc 100644 --- a/src/Fuzz/Manager.fs +++ b/src/Fuzz/Manager.fs @@ -10,9 +10,9 @@ let mutable testcaseDir = "" let mutable crashDir = "" let initialize outDir = - testcaseDir <- System.IO.Path.Combine(outDir, "testcase") + testcaseDir <- System.IO.Path.Combine(outDir, "queue") System.IO.Directory.CreateDirectory(testcaseDir) |> ignore - crashDir <- System.IO.Path.Combine(outDir, "crash") + crashDir <- System.IO.Path.Combine(outDir, "crashes") System.IO.Directory.CreateDirectory(crashDir) |> ignore (*** Statistics ***) @@ -35,8 +35,7 @@ let isNewPath pathHash = let private addPathHash pathHash = // The case of pathHash = 0UL means abortion due to threshold or other errors. - let isNewPath = pathHash <> 0UL && pathHashes.Add pathHash - isNewPath + pathHash <> 0UL && pathHashes.Add pathHash let private addCrashHash crashEdgeHash = crashEdgeHash <> 0UL && crashEdgeHashes.Add crashEdgeHash @@ -57,7 +56,7 @@ let printStatistics () = log " Floating point error : %d" fpErrorCount log " Program abortion : %d" abortCount -let private updateCrashCount seed exitSig = +let private updateCrashCount exitSig = match exitSig with | Signal.SIGSEGV -> segfaultCount <- segfaultCount + 1 | Signal.SIGILL -> illegalInstrCount <- illegalInstrCount + 1 @@ -66,7 +65,7 @@ let private updateCrashCount seed exitSig = | _ -> failwith "updateCrashCount() called with a non-crashing exit signal" crashCount <- crashCount + 1 -let private updateTestcaseCount seed = +let private updateTestcaseCount () = testCaseCount <- testCaseCount + 1 (*** Test case storing functions ***) @@ -76,13 +75,13 @@ let private dumpCrash opt seed exitSig = let crashName = sprintf "crash-%05d" crashCount let crashPath = System.IO.Path.Combine(crashDir, crashName) System.IO.File.WriteAllBytes(crashPath, Seed.concretize seed) - updateCrashCount seed exitSig + updateCrashCount exitSig let private dumpTestCase seed = let tcName = sprintf "tc-%05d" testCaseCount let tcPath = System.IO.Path.Combine(testcaseDir, tcName) System.IO.File.WriteAllBytes(tcPath, Seed.concretize seed) - updateTestcaseCount seed + updateTestcaseCount () let private checkCrash opt exitSig seed edgeHash = if Signal.isCrash exitSig && addCrashHash edgeHash diff --git a/src/Fuzz/SeedQueue.fs b/src/Fuzz/SeedQueue.fs index defe88b..029ae64 100644 --- a/src/Fuzz/SeedQueue.fs +++ b/src/Fuzz/SeedQueue.fs @@ -1,6 +1,41 @@ namespace Eclipser -type ConcolicQueue = { +/// A simple, purely functional queue. +type Queue<'a > = { + Enqueued : 'a list + ToDequeue : 'a list +} + +module Queue = + + exception EmptyException + + let empty = { Enqueued = []; ToDequeue = [] } + + let isEmpty q = List.isEmpty q.Enqueued && List.isEmpty q.ToDequeue + + /// Enqueue an element to a queue. + let enqueue q elem = { q with Enqueued = elem :: q.Enqueued } + + /// Dequeue an element from a queue. Raises Queue.EmptyException if the + /// queue is empty. + let dequeue q = + match q.Enqueued, q.ToDequeue with + | [], [] -> raise EmptyException + | enq, [] -> + let enqRev = List.rev enq + let elem, toDequeue = List.head enqRev, List.tail enqRev + (elem, { Enqueued = []; ToDequeue = toDequeue }) + | enq, deqHd :: deqTail -> + (deqHd, { Enqueued = enq; ToDequeue = deqTail }) + + let toString stringfy q = + let elems = q.ToDequeue @ (List.rev q.Enqueued) + let elemStrs = List.map stringfy elems + String.concat "\n" elemStrs + +/// Queue of seeds with 'favored' priority and 'normal' priority. +type SeedQueue = { Favoreds : Queue Normals : Queue } diff --git a/src/Fuzz/GreyConcolic/BranchTrace.fs b/src/GreyConcolic/BranchTrace.fs similarity index 95% rename from src/Fuzz/GreyConcolic/BranchTrace.fs rename to src/GreyConcolic/BranchTrace.fs index 3a5630c..b6f90d5 100644 --- a/src/Fuzz/GreyConcolic/BranchTrace.fs +++ b/src/GreyConcolic/BranchTrace.fs @@ -10,12 +10,12 @@ module BranchTrace = let collectAux opt (accTraces, accNewPathSeeds, accPaths) seed v = let pathHash, branchTrace = Executor.getBranchTrace opt seed v let accTraces = branchTrace :: accTraces - let accCandidates = + let accNewPathSeeds = if Manager.isNewPath pathHash && not (Set.contains pathHash accPaths) then seed :: accNewPathSeeds else accNewPathSeeds let accPaths = Set.add pathHash accPaths - (accTraces, accCandidates, accPaths) + (accTraces, accNewPathSeeds, accPaths) let collect seed opt minVal maxVal = let nSpawn = opt.NSpawn diff --git a/src/Fuzz/GreyConcolic/BranchTree.fs b/src/GreyConcolic/BranchTree.fs similarity index 100% rename from src/Fuzz/GreyConcolic/BranchTree.fs rename to src/GreyConcolic/BranchTree.fs diff --git a/src/GreyConcolic/GreyConcolic.fs b/src/GreyConcolic/GreyConcolic.fs new file mode 100644 index 0000000..c1465de --- /dev/null +++ b/src/GreyConcolic/GreyConcolic.fs @@ -0,0 +1,18 @@ +module Eclipser.GreyConcolic + +let run seed opt = + let curByteVal = Seed.getCurByteVal seed + let minByte, maxByte = ByteVal.getMinMax curByteVal seed.Source + if minByte = maxByte then + let seedStr = Seed.toString seed + failwithf "Cursor pointing to Fixed ByteVal %s" seedStr + let minVal, maxVal = bigint (int minByte), bigint (int maxByte) + let branchTraces, spawnSeeds = BranchTrace.collect seed opt minVal maxVal + let byteDir = Seed.getByteCursorDir seed + let bytes = Seed.queryNeighborBytes seed byteDir + let ctx = { Bytes = bytes; ByteDir = byteDir } + let branchTree = BranchTree.make opt ctx branchTraces + let branchTree = BranchTree.selectAndRepair opt branchTree + GreySolver.clearSolutionCache () + let solutions = GreySolver.solve seed opt byteDir branchTree + solutions @ spawnSeeds diff --git a/src/Fuzz/GreyConcolic/LinearEquation.fs b/src/GreyConcolic/LinearEquation.fs similarity index 100% rename from src/Fuzz/GreyConcolic/LinearEquation.fs rename to src/GreyConcolic/LinearEquation.fs diff --git a/src/Fuzz/GreyConcolic/LinearInequality.fs b/src/GreyConcolic/LinearInequality.fs similarity index 100% rename from src/Fuzz/GreyConcolic/LinearInequality.fs rename to src/GreyConcolic/LinearInequality.fs diff --git a/src/Fuzz/GreyConcolic/Linearity.fs b/src/GreyConcolic/Linearity.fs similarity index 100% rename from src/Fuzz/GreyConcolic/Linearity.fs rename to src/GreyConcolic/Linearity.fs diff --git a/src/Fuzz/GreyConcolic/Monotonicity.fs b/src/GreyConcolic/Monotonicity.fs similarity index 100% rename from src/Fuzz/GreyConcolic/Monotonicity.fs rename to src/GreyConcolic/Monotonicity.fs diff --git a/src/Fuzz/GreyConcolic/PathConstraint.fs b/src/GreyConcolic/PathConstraint.fs similarity index 100% rename from src/Fuzz/GreyConcolic/PathConstraint.fs rename to src/GreyConcolic/PathConstraint.fs diff --git a/src/Fuzz/GreyConcolic/Solve.fs b/src/GreyConcolic/Solve.fs similarity index 100% rename from src/Fuzz/GreyConcolic/Solve.fs rename to src/GreyConcolic/Solve.fs From 7cc6187f4b7aded2ca58c1e6f69da862d2b3fcc7 Mon Sep 17 00:00:00 2001 From: Jaeseung Choi Date: Mon, 21 Sep 2020 23:07:20 -0700 Subject: [PATCH 05/29] Cleanup QEMU instrumentation code Remove unused instrumentation module (i.e. system call tracing), and rename each tracer into more appropriate names (e.g. from 'pathcov' to 'coverage'). --- Instrumentor/qemu/build_qemu_x64.sh | 16 +- Instrumentor/qemu/build_qemu_x86.sh | 16 +- Instrumentor/qemu/gen_branch_patch.sh | 32 +++ Instrumentor/qemu/gen_coverage_patch.sh | 22 ++ Instrumentor/qemu/gen_feedback_patch.sh | 32 --- Instrumentor/qemu/gen_pathcov_patch.sh | 22 -- Instrumentor/qemu/gen_syscall_patch.sh | 12 - .../afl-qemu-cpu-inl.h | 0 .../chatkey.c | 46 ++-- .../cpu-exec.diff | 8 +- .../makefile-target.diff | 4 +- .../optimize.diff | 4 +- .../syscall.diff | 4 +- .../tcg-op.diff | 4 +- .../tcg-opc.diff | 4 +- .../tcg-target.diff | 4 +- .../tcg.diff | 4 +- .../translate.diff | 4 +- .../afl-qemu-cpu-inl.h | 3 +- .../chatkey.cc | 93 +------ .../cpu-exec.diff | 22 +- .../exec-all.diff | 4 +- .../makefile-target.diff | 4 +- .../syscall.diff | 4 +- .../translate.diff | 4 +- .../qemu/patches-pathcov/chatkey-utils.h | 50 ---- Instrumentor/qemu/patches-syscall/chatkey.c | 144 ---------- .../qemu/patches-syscall/cpu-exec.diff | 24 -- .../qemu/patches-syscall/makefile-objs.diff | 9 - .../qemu/patches-syscall/syscall.diff | 256 ------------------ Instrumentor/qemu/prepare_qemu.sh | 89 +++--- Instrumentor/qemu/rebuild.sh | 32 +-- Instrumentor/qemu/repatch.sh | 178 +++++------- src/Core/Executor.fs | 19 +- src/Core/libexec.c | 10 +- 35 files changed, 266 insertions(+), 917 deletions(-) create mode 100755 Instrumentor/qemu/gen_branch_patch.sh create mode 100755 Instrumentor/qemu/gen_coverage_patch.sh delete mode 100755 Instrumentor/qemu/gen_feedback_patch.sh delete mode 100755 Instrumentor/qemu/gen_pathcov_patch.sh delete mode 100755 Instrumentor/qemu/gen_syscall_patch.sh rename Instrumentor/qemu/{patches-feedback => patches-branch}/afl-qemu-cpu-inl.h (100%) rename Instrumentor/qemu/{patches-feedback => patches-branch}/chatkey.c (88%) rename Instrumentor/qemu/{patches-feedback => patches-branch}/cpu-exec.diff (91%) rename Instrumentor/qemu/{patches-feedback => patches-branch}/makefile-target.diff (69%) rename Instrumentor/qemu/{patches-feedback => patches-branch}/optimize.diff (96%) rename Instrumentor/qemu/{patches-pathcov => patches-branch}/syscall.diff (98%) rename Instrumentor/qemu/{patches-feedback => patches-branch}/tcg-op.diff (97%) rename Instrumentor/qemu/{patches-feedback => patches-branch}/tcg-opc.diff (89%) rename Instrumentor/qemu/{patches-feedback => patches-branch}/tcg-target.diff (96%) rename Instrumentor/qemu/{patches-feedback => patches-branch}/tcg.diff (87%) rename Instrumentor/qemu/{patches-feedback => patches-branch}/translate.diff (99%) rename Instrumentor/qemu/{patches-pathcov => patches-coverage}/afl-qemu-cpu-inl.h (97%) rename Instrumentor/qemu/{patches-pathcov => patches-coverage}/chatkey.cc (60%) rename Instrumentor/qemu/{patches-pathcov => patches-coverage}/cpu-exec.diff (71%) rename Instrumentor/qemu/{patches-pathcov => patches-coverage}/exec-all.diff (59%) rename Instrumentor/qemu/{patches-pathcov => patches-coverage}/makefile-target.diff (80%) rename Instrumentor/qemu/{patches-feedback => patches-coverage}/syscall.diff (98%) rename Instrumentor/qemu/{patches-pathcov => patches-coverage}/translate.diff (97%) delete mode 100644 Instrumentor/qemu/patches-pathcov/chatkey-utils.h delete mode 100644 Instrumentor/qemu/patches-syscall/chatkey.c delete mode 100644 Instrumentor/qemu/patches-syscall/cpu-exec.diff delete mode 100644 Instrumentor/qemu/patches-syscall/makefile-objs.diff delete mode 100644 Instrumentor/qemu/patches-syscall/syscall.diff diff --git a/Instrumentor/qemu/build_qemu_x64.sh b/Instrumentor/qemu/build_qemu_x64.sh index a06468b..97be8f8 100755 --- a/Instrumentor/qemu/build_qemu_x64.sh +++ b/Instrumentor/qemu/build_qemu_x64.sh @@ -40,17 +40,13 @@ build_qemu () { ### Build QEMU tracers -build_qemu pathcov -mv "./qemu-trace" "../../build/qemu-trace-pathcov-x64" || exit 1 -echo "[+] Successfully created 'qemu-trace-pathcov-x64'." +build_qemu coverage +mv "./qemu-trace" "../../build/qemu-trace-coverage-x64" || exit 1 +echo "[+] Successfully created 'qemu-trace-coverage-x64'." -build_qemu syscall -mv "./qemu-trace" "../../build/qemu-trace-syscall-x64" || exit 1 -echo "[+] Successfully created 'qemu-trace-syscall-x64'." - -build_qemu feedback -mv "./qemu-trace" "../../build/qemu-trace-feedback-x64" || exit 1 -echo "[+] Successfully created 'qemu-trace-feedback-x64'." +build_qemu branch +mv "./qemu-trace" "../../build/qemu-trace-branch-x64" || exit 1 +echo "[+] Successfully created 'qemu-trace-branch-x64'." build_qemu bbcount mv "./qemu-trace" "../../build/qemu-trace-bbcount-x64" || exit 1 diff --git a/Instrumentor/qemu/build_qemu_x86.sh b/Instrumentor/qemu/build_qemu_x86.sh index 278f16f..50a57ab 100755 --- a/Instrumentor/qemu/build_qemu_x86.sh +++ b/Instrumentor/qemu/build_qemu_x86.sh @@ -40,17 +40,13 @@ build_qemu () { ### Build QEMU tracers -build_qemu pathcov -mv "./qemu-trace" "../../build/qemu-trace-pathcov-x86" || exit 1 -echo "[+] Successfully created 'qemu-trace-pathcov-x86'." +build_qemu coverage +mv "./qemu-trace" "../../build/qemu-trace-coverage-x86" || exit 1 +echo "[+] Successfully created 'qemu-trace-coverage-x86'." -build_qemu syscall -mv "./qemu-trace" "../../build/qemu-trace-syscall-x86" || exit 1 -echo "[+] Successfully created 'qemu-trace-syscall-x86'." - -build_qemu feedback -mv "./qemu-trace" "../../build/qemu-trace-feedback-x86" || exit 1 -echo "[+] Successfully created 'qemu-trace-feedback-x86'." +build_qemu branch +mv "./qemu-trace" "../../build/qemu-trace-branch-x86" || exit 1 +echo "[+] Successfully created 'qemu-trace-branch-x86'." build_qemu bbcount mv "./qemu-trace" "../../build/qemu-trace-bbcount-x86" || exit 1 diff --git a/Instrumentor/qemu/gen_branch_patch.sh b/Instrumentor/qemu/gen_branch_patch.sh new file mode 100755 index 0000000..02b9bfb --- /dev/null +++ b/Instrumentor/qemu/gen_branch_patch.sh @@ -0,0 +1,32 @@ +#!/bin/bash + +cp qemu-2.3.0-branch/tcg/chatkey.c ./patches-branch/chatkey.c + +cp qemu-2.3.0-branch/afl-qemu-cpu-inl.h ./patches-branch/afl-qemu-cpu-inl.h + +cp qemu-2.3.0/target-i386/translate.c qemu-2.3.0-branch/target-i386/translate.c.orig +diff -Naur qemu-2.3.0-branch/target-i386/translate.c.orig qemu-2.3.0-branch/target-i386/translate.c > patches-branch/translate.diff + +cp qemu-2.3.0/cpu-exec.c qemu-2.3.0-branch/cpu-exec.c.orig +diff -Naur qemu-2.3.0-branch/cpu-exec.c.orig qemu-2.3.0-branch/cpu-exec.c > patches-branch/cpu-exec.diff + +cp qemu-2.3.0/linux-user/syscall.c qemu-2.3.0-branch/linux-user/syscall.c.orig +diff -Naur qemu-2.3.0-branch/linux-user/syscall.c.orig qemu-2.3.0-branch/linux-user/syscall.c > patches-branch/syscall.diff + +cp qemu-2.3.0/Makefile.target qemu-2.3.0-branch/Makefile.target.orig +diff -Naur qemu-2.3.0-branch/Makefile.target.orig qemu-2.3.0-branch/Makefile.target > patches-branch/makefile-target.diff + +cp qemu-2.3.0/tcg/tcg-opc.h qemu-2.3.0-branch/tcg/tcg-opc.h.orig +diff -Naur qemu-2.3.0-branch/tcg/tcg-opc.h.orig qemu-2.3.0-branch/tcg/tcg-opc.h > patches-branch/tcg-opc.diff + +cp qemu-2.3.0/tcg/tcg-op.h qemu-2.3.0-branch/tcg/tcg-op.h.orig +diff -Naur qemu-2.3.0-branch/tcg/tcg-op.h.orig qemu-2.3.0-branch/tcg/tcg-op.h > patches-branch/tcg-op.diff + +cp qemu-2.3.0/tcg/i386/tcg-target.c qemu-2.3.0-branch/tcg/i386/tcg-target.c.orig +diff -Naur qemu-2.3.0-branch/tcg/i386/tcg-target.c.orig qemu-2.3.0-branch/tcg/i386/tcg-target.c > patches-branch/tcg-target.diff + +cp qemu-2.3.0/tcg/tcg.h qemu-2.3.0-branch/tcg/tcg.h.orig +diff -Naur qemu-2.3.0-branch/tcg/tcg.h.orig qemu-2.3.0-branch/tcg/tcg.h > patches-branch/tcg.diff + +cp qemu-2.3.0/tcg/optimize.c qemu-2.3.0-branch/tcg/optimize.c.orig +diff -Naur qemu-2.3.0-branch/tcg/optimize.c.orig qemu-2.3.0-branch/tcg/optimize.c > patches-branch/optimize.diff diff --git a/Instrumentor/qemu/gen_coverage_patch.sh b/Instrumentor/qemu/gen_coverage_patch.sh new file mode 100755 index 0000000..16bff56 --- /dev/null +++ b/Instrumentor/qemu/gen_coverage_patch.sh @@ -0,0 +1,22 @@ +#!/bin/bash + +cp qemu-2.3.0-coverage/chatkey.cc ./patches-coverage/chatkey.cc + +cp qemu-2.3.0-coverage/afl-qemu-cpu-inl.h ./patches-coverage/afl-qemu-cpu-inl.h + +cp qemu-2.3.0-coverage/chatkey-utils.h ./patches-coverage/chatkey-utils.h + +cp qemu-2.3.0/linux-user/syscall.c qemu-2.3.0-coverage/linux-user/syscall.c.orig +diff -Naur qemu-2.3.0-coverage/linux-user/syscall.c.orig qemu-2.3.0-coverage/linux-user/syscall.c > patches-coverage/syscall.diff + +cp qemu-2.3.0/target-i386/translate.c qemu-2.3.0-coverage/target-i386/translate.c.orig +diff -Naur qemu-2.3.0-coverage/target-i386/translate.c.orig qemu-2.3.0-coverage/target-i386/translate.c > patches-coverage/translate.diff + +cp qemu-2.3.0/cpu-exec.c qemu-2.3.0-coverage/cpu-exec.c.orig +diff -Naur qemu-2.3.0-coverage/cpu-exec.c.orig qemu-2.3.0-coverage/cpu-exec.c > patches-coverage/cpu-exec.diff + +cp qemu-2.3.0/Makefile.target qemu-2.3.0-coverage/Makefile.target.orig +diff -Naur qemu-2.3.0-coverage/Makefile.target.orig qemu-2.3.0-coverage/Makefile.target > patches-coverage/makefile-target.diff + +cp qemu-2.3.0/include/exec/exec-all.h ./qemu-2.3.0-coverage/include/exec/exec-all.h.orig +diff -Naur qemu-2.3.0-coverage/include/exec/exec-all.h.orig qemu-2.3.0-coverage/include/exec/exec-all.h > patches-coverage/exec-all.diff diff --git a/Instrumentor/qemu/gen_feedback_patch.sh b/Instrumentor/qemu/gen_feedback_patch.sh deleted file mode 100755 index 584de03..0000000 --- a/Instrumentor/qemu/gen_feedback_patch.sh +++ /dev/null @@ -1,32 +0,0 @@ -#!/bin/bash - -cp qemu-2.3.0-feedback/tcg/chatkey.c ./patches-feedback/chatkey.c - -cp qemu-2.3.0-feedback/afl-qemu-cpu-inl.h ./patches-feedback/afl-qemu-cpu-inl.h - -cp qemu-2.3.0/target-i386/translate.c qemu-2.3.0-feedback/target-i386/translate.c.orig -diff -Naur qemu-2.3.0-feedback/target-i386/translate.c.orig qemu-2.3.0-feedback/target-i386/translate.c > patches-feedback/translate.diff - -cp qemu-2.3.0/cpu-exec.c qemu-2.3.0-feedback/cpu-exec.c.orig -diff -Naur qemu-2.3.0-feedback/cpu-exec.c.orig qemu-2.3.0-feedback/cpu-exec.c > patches-feedback/cpu-exec.diff - -cp qemu-2.3.0/linux-user/syscall.c qemu-2.3.0-feedback/linux-user/syscall.c.orig -diff -Naur qemu-2.3.0-feedback/linux-user/syscall.c.orig qemu-2.3.0-feedback/linux-user/syscall.c > patches-feedback/syscall.diff - -cp qemu-2.3.0/Makefile.target qemu-2.3.0-feedback/Makefile.target.orig -diff -Naur qemu-2.3.0-feedback/Makefile.target.orig qemu-2.3.0-feedback/Makefile.target > patches-feedback/makefile-target.diff - -cp qemu-2.3.0/tcg/tcg-opc.h qemu-2.3.0-feedback/tcg/tcg-opc.h.orig -diff -Naur qemu-2.3.0-feedback/tcg/tcg-opc.h.orig qemu-2.3.0-feedback/tcg/tcg-opc.h > patches-feedback/tcg-opc.diff - -cp qemu-2.3.0/tcg/tcg-op.h qemu-2.3.0-feedback/tcg/tcg-op.h.orig -diff -Naur qemu-2.3.0-feedback/tcg/tcg-op.h.orig qemu-2.3.0-feedback/tcg/tcg-op.h > patches-feedback/tcg-op.diff - -cp qemu-2.3.0/tcg/i386/tcg-target.c qemu-2.3.0-feedback/tcg/i386/tcg-target.c.orig -diff -Naur qemu-2.3.0-feedback/tcg/i386/tcg-target.c.orig qemu-2.3.0-feedback/tcg/i386/tcg-target.c > patches-feedback/tcg-target.diff - -cp qemu-2.3.0/tcg/tcg.h qemu-2.3.0-feedback/tcg/tcg.h.orig -diff -Naur qemu-2.3.0-feedback/tcg/tcg.h.orig qemu-2.3.0-feedback/tcg/tcg.h > patches-feedback/tcg.diff - -cp qemu-2.3.0/tcg/optimize.c qemu-2.3.0-feedback/tcg/optimize.c.orig -diff -Naur qemu-2.3.0-feedback/tcg/optimize.c.orig qemu-2.3.0-feedback/tcg/optimize.c > patches-feedback/optimize.diff diff --git a/Instrumentor/qemu/gen_pathcov_patch.sh b/Instrumentor/qemu/gen_pathcov_patch.sh deleted file mode 100755 index b9032c8..0000000 --- a/Instrumentor/qemu/gen_pathcov_patch.sh +++ /dev/null @@ -1,22 +0,0 @@ -#!/bin/bash - -cp qemu-2.3.0-pathcov/chatkey.cc ./patches-pathcov/chatkey.cc - -cp qemu-2.3.0-pathcov/afl-qemu-cpu-inl.h ./patches-pathcov/afl-qemu-cpu-inl.h - -cp qemu-2.3.0-pathcov/chatkey-utils.h ./patches-pathcov/chatkey-utils.h - -cp qemu-2.3.0/linux-user/syscall.c qemu-2.3.0-pathcov/linux-user/syscall.c.orig -diff -Naur qemu-2.3.0-pathcov/linux-user/syscall.c.orig qemu-2.3.0-pathcov/linux-user/syscall.c > patches-pathcov/syscall.diff - -cp qemu-2.3.0/target-i386/translate.c qemu-2.3.0-pathcov/target-i386/translate.c.orig -diff -Naur qemu-2.3.0-pathcov/target-i386/translate.c.orig qemu-2.3.0-pathcov/target-i386/translate.c > patches-pathcov/translate.diff - -cp qemu-2.3.0/cpu-exec.c qemu-2.3.0-pathcov/cpu-exec.c.orig -diff -Naur qemu-2.3.0-pathcov/cpu-exec.c.orig qemu-2.3.0-pathcov/cpu-exec.c > patches-pathcov/cpu-exec.diff - -cp qemu-2.3.0/Makefile.target qemu-2.3.0-pathcov/Makefile.target.orig -diff -Naur qemu-2.3.0-pathcov/Makefile.target.orig qemu-2.3.0-pathcov/Makefile.target > patches-pathcov/makefile-target.diff - -cp qemu-2.3.0/include/exec/exec-all.h ./qemu-2.3.0-pathcov/include/exec/exec-all.h.orig -diff -Naur qemu-2.3.0-pathcov/include/exec/exec-all.h.orig qemu-2.3.0-pathcov/include/exec/exec-all.h > patches-pathcov/exec-all.diff diff --git a/Instrumentor/qemu/gen_syscall_patch.sh b/Instrumentor/qemu/gen_syscall_patch.sh deleted file mode 100755 index 136d371..0000000 --- a/Instrumentor/qemu/gen_syscall_patch.sh +++ /dev/null @@ -1,12 +0,0 @@ -#!/bin/bash - -cp qemu-2.3.0-syscall/linux-user/chatkey.c ./patches-syscall/chatkey.c - -cp qemu-2.3.0/linux-user/syscall.c qemu-2.3.0-syscall/linux-user/syscall.c.orig -diff -Naur qemu-2.3.0-syscall/linux-user/syscall.c.orig qemu-2.3.0-syscall/linux-user/syscall.c > patches-syscall/syscall.diff - -cp qemu-2.3.0/cpu-exec.c qemu-2.3.0-syscall/cpu-exec.c.orig -diff -Naur qemu-2.3.0-syscall/cpu-exec.c.orig qemu-2.3.0-syscall/cpu-exec.c > patches-syscall/cpu-exec.diff - -cp qemu-2.3.0/linux-user/Makefile.objs qemu-2.3.0-syscall/linux-user/Makefile.objs.orig -diff -Naur qemu-2.3.0-syscall/linux-user/Makefile.objs.orig qemu-2.3.0-syscall/linux-user/Makefile.objs > patches-syscall/makefile-objs.diff diff --git a/Instrumentor/qemu/patches-feedback/afl-qemu-cpu-inl.h b/Instrumentor/qemu/patches-branch/afl-qemu-cpu-inl.h similarity index 100% rename from Instrumentor/qemu/patches-feedback/afl-qemu-cpu-inl.h rename to Instrumentor/qemu/patches-branch/afl-qemu-cpu-inl.h diff --git a/Instrumentor/qemu/patches-feedback/chatkey.c b/Instrumentor/qemu/patches-branch/chatkey.c similarity index 88% rename from Instrumentor/qemu/patches-feedback/chatkey.c rename to Instrumentor/qemu/patches-branch/chatkey.c index 3be2bcc..07e9451 100644 --- a/Instrumentor/qemu/patches-feedback/chatkey.c +++ b/Instrumentor/qemu/patches-branch/chatkey.c @@ -33,29 +33,29 @@ bool chatkey_EP_passed = false; static size_t targ_hit_count = 0; static size_t trace_count = 0; -static FILE * log_fp; +static FILE * branch_fp; static FILE * hash_fp; void flush_trace_buffer(void); void chatkey_setup(void); void chatkey_close_fp(void); void chatkey_exit(void); -void chatkey_log_feedback(abi_ulong oprnd1, abi_ulong oprnd2, unsigned char operand_type); +void chatkey_log_branch(abi_ulong oprnd1, abi_ulong oprnd2, unsigned char operand_type); void chatkey_update_hash(register abi_ulong addr); void flush_trace_buffer(void) { size_t len = buf_ptr - trace_buffer; - fwrite(trace_buffer, len, 1, log_fp); + fwrite(trace_buffer, len, 1, branch_fp); } void chatkey_setup(void) { - char * log_path = getenv("CK_FEED_LOG"); + char * branch_path = getenv("CK_FEED_LOG"); char * hash_path = getenv("CK_HASH_LOG"); - assert(log_path != NULL); - log_fp = fopen(log_path, "w"); - assert(log_fp != NULL); + assert(branch_path != NULL); + branch_fp = fopen(branch_path, "w"); + assert(branch_fp != NULL); assert(hash_path != NULL); hash_fp = fopen(hash_path, "w"); @@ -73,14 +73,14 @@ void chatkey_setup(void) { // When fork() syscall is encountered, child process should call this function void chatkey_close_fp(void) { - // close 'log_fp', since we don't want to dump log twice - fclose(log_fp); - log_fp = NULL; + // close 'branch_fp', since we don't want to dump log twice + fclose(branch_fp); + branch_fp = NULL; fclose(hash_fp); if (afl_forksrv_pid) - close(TSL_FD); + close(TSL_FD); } void chatkey_exit(void) { @@ -88,7 +88,7 @@ void chatkey_exit(void) { sigset_t mask; // If chatkey_close_fp() was called, then return without any action - if (log_fp == NULL) + if (branch_fp == NULL) return; // Block signals, since we register signal handler that calls chatkey_exit() @@ -99,8 +99,8 @@ void chatkey_exit(void) { flush_trace_buffer(); - fwrite(&nil, sizeof(abi_ulong), 1, log_fp); - fclose(log_fp); + fwrite(&nil, sizeof(abi_ulong), 1, branch_fp); + fclose(branch_fp); #ifdef TARGET_X86_64 fprintf(hash_fp, "%lu\n", hash); @@ -122,7 +122,7 @@ asm (".global chatkey_trampoline \t\n\ push %r9 \t\n\ push %r10 \t\n\ push %r11 \t\n\ - call chatkey_log_feedback; \t\n\ + call chatkey_log_branch; \t\n\ pop %r11 \t\n\ pop %r10 \t\n\ pop %r9 \t\n\ @@ -133,14 +133,14 @@ asm (".global chatkey_trampoline \t\n\ .size chatkey_trampoline, . - chatkey_trampoline \t\n\ "); -void chatkey_log_feedback(abi_ulong oprnd1, abi_ulong oprnd2, unsigned char type) { +void chatkey_log_branch(abi_ulong oprnd1, abi_ulong oprnd2, unsigned char type) { abi_ulong oprnd1_truncated, oprnd2_truncated; unsigned char operand_type = type & 0x3f; unsigned char compare_type = type & 0xc0; unsigned char operand_size; - if (!log_fp) + if (!branch_fp) return; if (chatkey_targ_addr) { @@ -178,10 +178,10 @@ void chatkey_log_feedback(abi_ulong oprnd1, abi_ulong oprnd2, unsigned char type } type = compare_type | operand_size; - fwrite(&chatkey_curr_addr, sizeof(abi_ulong), 1, log_fp); - fwrite(&type, sizeof(unsigned char), 1, log_fp); - fwrite(&oprnd1_truncated, operand_size, 1, log_fp); - fwrite(&oprnd2_truncated, operand_size, 1, log_fp); + fwrite(&chatkey_curr_addr, sizeof(abi_ulong), 1, branch_fp); + fwrite(&type, sizeof(unsigned char), 1, branch_fp); + fwrite(&oprnd1_truncated, operand_size, 1, branch_fp); + fwrite(&oprnd2_truncated, operand_size, 1, branch_fp); if (oprnd1_truncated != oprnd2_truncated) { /* If two operands are not equal, then path hash of this execution is * not used in Chatkey. Therefore, finish execution to save time. @@ -243,8 +243,8 @@ void chatkey_log_feedback(abi_ulong oprnd1, abi_ulong oprnd2, unsigned char type * limit has exceeded. Abort tracing. */ abi_ulong nil = 0; flush_trace_buffer(); - fwrite(&nil, sizeof(abi_ulong), 1, log_fp); - fclose(log_fp); + fwrite(&nil, sizeof(abi_ulong), 1, branch_fp); + fclose(branch_fp); // output 0 as path hash to indicate abortion. fprintf(hash_fp, "0\n"); fclose(hash_fp); diff --git a/Instrumentor/qemu/patches-feedback/cpu-exec.diff b/Instrumentor/qemu/patches-branch/cpu-exec.diff similarity index 91% rename from Instrumentor/qemu/patches-feedback/cpu-exec.diff rename to Instrumentor/qemu/patches-branch/cpu-exec.diff index d9f18a6..3186a77 100644 --- a/Instrumentor/qemu/patches-feedback/cpu-exec.diff +++ b/Instrumentor/qemu/patches-branch/cpu-exec.diff @@ -1,5 +1,5 @@ ---- qemu-2.3.0-feedback/cpu-exec.c.orig 2017-09-17 01:29:39.966426739 +0900 -+++ qemu-2.3.0-feedback/cpu-exec.c 2017-09-17 01:28:53.506606640 +0900 +--- qemu-2.3.0-branch/cpu-exec.c.orig 2017-09-17 01:29:39.966426739 +0900 ++++ qemu-2.3.0-branch/cpu-exec.c 2017-09-17 01:28:53.506606640 +0900 @@ -28,6 +28,14 @@ #include "exec/memory-internal.h" #include "qemu/rcu.h" @@ -40,8 +40,8 @@ + if(tb->pc == chatkey_entry_point) { + /* XXX. If the following line is moved into chatkey_setup(), a + * weired bug occurs when fork server is enabled. In the first -+ * execution, execution feedbacks are correcly collected. But -+ * in the subsequent executions, chatkey_log_feedback() is not ++ * execution, execution branchs are correcly collected. But ++ * in the subsequent executions, chatkey_log_branch() is not + * called at all. + */ + chatkey_EP_passed = true; diff --git a/Instrumentor/qemu/patches-feedback/makefile-target.diff b/Instrumentor/qemu/patches-branch/makefile-target.diff similarity index 69% rename from Instrumentor/qemu/patches-feedback/makefile-target.diff rename to Instrumentor/qemu/patches-branch/makefile-target.diff index ac4b219..87b8e5e 100644 --- a/Instrumentor/qemu/patches-feedback/makefile-target.diff +++ b/Instrumentor/qemu/patches-branch/makefile-target.diff @@ -1,5 +1,5 @@ ---- qemu-2.3.0-feedback/Makefile.target.orig 2017-04-13 20:44:12.868483773 +0900 -+++ qemu-2.3.0-feedback/Makefile.target 2017-04-10 16:46:23.364441824 +0900 +--- qemu-2.3.0-branch/Makefile.target.orig 2017-04-13 20:44:12.868483773 +0900 ++++ qemu-2.3.0-branch/Makefile.target 2017-04-10 16:46:23.364441824 +0900 @@ -83,7 +83,7 @@ ######################################################### # cpu emulator library diff --git a/Instrumentor/qemu/patches-feedback/optimize.diff b/Instrumentor/qemu/patches-branch/optimize.diff similarity index 96% rename from Instrumentor/qemu/patches-feedback/optimize.diff rename to Instrumentor/qemu/patches-branch/optimize.diff index 0321827..e27e70d 100644 --- a/Instrumentor/qemu/patches-feedback/optimize.diff +++ b/Instrumentor/qemu/patches-branch/optimize.diff @@ -1,5 +1,5 @@ ---- qemu-2.3.0-feedback/tcg/optimize.c.orig 2017-08-22 16:45:31.562156045 +0900 -+++ qemu-2.3.0-feedback/tcg/optimize.c 2017-08-22 16:16:49.397284593 +0900 +--- qemu-2.3.0-branch/tcg/optimize.c.orig 2017-08-22 16:45:31.562156045 +0900 ++++ qemu-2.3.0-branch/tcg/optimize.c 2017-08-22 16:16:49.397284593 +0900 @@ -258,8 +258,8 @@ CASE_OP_32_64(add): return x + y; diff --git a/Instrumentor/qemu/patches-pathcov/syscall.diff b/Instrumentor/qemu/patches-branch/syscall.diff similarity index 98% rename from Instrumentor/qemu/patches-pathcov/syscall.diff rename to Instrumentor/qemu/patches-branch/syscall.diff index 4fc0977..ccb0eaa 100644 --- a/Instrumentor/qemu/patches-pathcov/syscall.diff +++ b/Instrumentor/qemu/patches-branch/syscall.diff @@ -1,5 +1,5 @@ ---- qemu-2.3.0-pathcov/linux-user/syscall.c.orig 2017-07-20 14:40:31.912793680 +0900 -+++ qemu-2.3.0-pathcov/linux-user/syscall.c 2017-07-20 14:25:16.001550285 +0900 +--- qemu-2.3.0-branch/linux-user/syscall.c.orig 2017-07-20 14:41:00.160771159 +0900 ++++ qemu-2.3.0-branch/linux-user/syscall.c 2017-07-20 14:34:26.127522059 +0900 @@ -115,6 +115,33 @@ #include "qemu.h" diff --git a/Instrumentor/qemu/patches-feedback/tcg-op.diff b/Instrumentor/qemu/patches-branch/tcg-op.diff similarity index 97% rename from Instrumentor/qemu/patches-feedback/tcg-op.diff rename to Instrumentor/qemu/patches-branch/tcg-op.diff index 48d45a0..9b9cb74 100644 --- a/Instrumentor/qemu/patches-feedback/tcg-op.diff +++ b/Instrumentor/qemu/patches-branch/tcg-op.diff @@ -1,5 +1,5 @@ ---- qemu-2.3.0-feedback/tcg/tcg-op.h.orig 2017-08-18 10:10:04.060465652 +0900 -+++ qemu-2.3.0-feedback/tcg/tcg-op.h 2017-08-18 10:08:03.058197082 +0900 +--- qemu-2.3.0-branch/tcg/tcg-op.h.orig 2017-08-18 10:10:04.060465652 +0900 ++++ qemu-2.3.0-branch/tcg/tcg-op.h 2017-08-18 10:08:03.058197082 +0900 @@ -381,14 +381,42 @@ tcg_gen_op3_i32(INDEX_op_add_i32, ret, arg1, arg2); } diff --git a/Instrumentor/qemu/patches-feedback/tcg-opc.diff b/Instrumentor/qemu/patches-branch/tcg-opc.diff similarity index 89% rename from Instrumentor/qemu/patches-feedback/tcg-opc.diff rename to Instrumentor/qemu/patches-branch/tcg-opc.diff index f023078..f631cb6 100644 --- a/Instrumentor/qemu/patches-feedback/tcg-opc.diff +++ b/Instrumentor/qemu/patches-branch/tcg-opc.diff @@ -1,5 +1,5 @@ ---- qemu-2.3.0-feedback/tcg/tcg-opc.h.orig 2017-04-13 20:44:12.868483773 +0900 -+++ qemu-2.3.0-feedback/tcg/tcg-opc.h 2017-04-10 16:46:23.372441850 +0900 +--- qemu-2.3.0-branch/tcg/tcg-opc.h.orig 2017-04-13 20:44:12.868483773 +0900 ++++ qemu-2.3.0-branch/tcg/tcg-opc.h 2017-04-10 16:46:23.372441850 +0900 @@ -57,7 +57,7 @@ DEF(st_i32, 0, 2, 1, 0) /* arith */ diff --git a/Instrumentor/qemu/patches-feedback/tcg-target.diff b/Instrumentor/qemu/patches-branch/tcg-target.diff similarity index 96% rename from Instrumentor/qemu/patches-feedback/tcg-target.diff rename to Instrumentor/qemu/patches-branch/tcg-target.diff index cd45943..c89fdaf 100644 --- a/Instrumentor/qemu/patches-feedback/tcg-target.diff +++ b/Instrumentor/qemu/patches-branch/tcg-target.diff @@ -1,5 +1,5 @@ ---- qemu-2.3.0-feedback/tcg/i386/tcg-target.c.orig 2017-10-01 21:59:22.790565093 +0900 -+++ qemu-2.3.0-feedback/tcg/i386/tcg-target.c 2017-10-01 21:57:46.366637219 +0900 +--- qemu-2.3.0-branch/tcg/i386/tcg-target.c.orig 2017-10-01 21:59:22.790565093 +0900 ++++ qemu-2.3.0-branch/tcg/i386/tcg-target.c 2017-10-01 21:57:46.366637219 +0900 @@ -24,6 +24,8 @@ #include "tcg-be-ldst.h" diff --git a/Instrumentor/qemu/patches-feedback/tcg.diff b/Instrumentor/qemu/patches-branch/tcg.diff similarity index 87% rename from Instrumentor/qemu/patches-feedback/tcg.diff rename to Instrumentor/qemu/patches-branch/tcg.diff index 7fe76a2..fe3dc18 100644 --- a/Instrumentor/qemu/patches-feedback/tcg.diff +++ b/Instrumentor/qemu/patches-branch/tcg.diff @@ -1,5 +1,5 @@ ---- qemu-2.3.0-feedback/tcg/tcg.h.orig 2017-08-18 10:12:58.561955139 +0900 -+++ qemu-2.3.0-feedback/tcg/tcg.h 2017-08-18 10:12:44.066164190 +0900 +--- qemu-2.3.0-branch/tcg/tcg.h.orig 2017-08-18 10:12:58.561955139 +0900 ++++ qemu-2.3.0-branch/tcg/tcg.h 2017-08-18 10:12:44.066164190 +0900 @@ -29,6 +29,15 @@ #include "qemu/bitops.h" #include "tcg-target.h" diff --git a/Instrumentor/qemu/patches-feedback/translate.diff b/Instrumentor/qemu/patches-branch/translate.diff similarity index 99% rename from Instrumentor/qemu/patches-feedback/translate.diff rename to Instrumentor/qemu/patches-branch/translate.diff index 11259d5..d5806ab 100644 --- a/Instrumentor/qemu/patches-feedback/translate.diff +++ b/Instrumentor/qemu/patches-branch/translate.diff @@ -1,5 +1,5 @@ ---- qemu-2.3.0-feedback/target-i386/translate.c.orig 2017-09-17 00:03:03.803748021 +0900 -+++ qemu-2.3.0-feedback/target-i386/translate.c 2017-09-16 23:51:44.773721311 +0900 +--- qemu-2.3.0-branch/target-i386/translate.c.orig 2017-09-17 00:03:03.803748021 +0900 ++++ qemu-2.3.0-branch/target-i386/translate.c 2017-09-16 23:51:44.773721311 +0900 @@ -60,6 +60,13 @@ # define clztl clz32 #endif diff --git a/Instrumentor/qemu/patches-pathcov/afl-qemu-cpu-inl.h b/Instrumentor/qemu/patches-coverage/afl-qemu-cpu-inl.h similarity index 97% rename from Instrumentor/qemu/patches-pathcov/afl-qemu-cpu-inl.h rename to Instrumentor/qemu/patches-coverage/afl-qemu-cpu-inl.h index 7cb7e0f..47ba5ae 100644 --- a/Instrumentor/qemu/patches-pathcov/afl-qemu-cpu-inl.h +++ b/Instrumentor/qemu/patches-coverage/afl-qemu-cpu-inl.h @@ -23,7 +23,6 @@ #define FORKSRV_FD 198 #define TSL_FD (FORKSRV_FD - 1) -extern int chatkey_mode; extern abi_ulong mmap_next_start; /* Set in the child process in forkserver mode: */ @@ -79,7 +78,7 @@ static void afl_forkserver(CPUArchState *env) { /* Whoops, parent dead? */ - if (read(FORKSRV_FD, &chatkey_mode, 4) != 4) exit(2); + if (read(FORKSRV_FD, &tmp, 4) != 4) exit(2); /* Establish a channel with child to grab translation commands. We'll read from t_fd[0], child will write to TSL_FD. */ diff --git a/Instrumentor/qemu/patches-pathcov/chatkey.cc b/Instrumentor/qemu/patches-coverage/chatkey.cc similarity index 60% rename from Instrumentor/qemu/patches-pathcov/chatkey.cc rename to Instrumentor/qemu/patches-coverage/chatkey.cc index de2f05e..3fc5f43 100644 --- a/Instrumentor/qemu/patches-pathcov/chatkey.cc +++ b/Instrumentor/qemu/patches-coverage/chatkey.cc @@ -21,38 +21,19 @@ extern unsigned int afl_forksrv_pid; abi_ulong chatkey_entry_point; /* ELF entry point (_start) */ -#define MODE_COUNT_NEW 0 // Count newly covered edges, along with path hash. -#define MODE_HASH 1 // Calculate edge set hash. -#define MODE_SET 2 // Return the set of the visited edges. -int chatkey_mode = -1; - +int passed_forkserver = 0; static uint32_t new_edge_cnt = 0; // # of new edges visited in this execution static abi_ulong edge_set_hash = 5381; // djb2 hash static abi_ulong path_hash = 5381; // djb2 hash static abi_ulong prev_node = 0; -/* Global file pointers */ static FILE* coverage_fp; static FILE* dbg_fp; -static int is_fp_closed = 0; - static unsigned char * accum_edge_bitmap; static unsigned char edge_bitmap[0x10000]; // Holds edges visited in this exec (will be dumped into a file) static dense_hash_set edge_set; -static void dump_set(dense_hash_set * set, FILE* output_fp) -{ - google::dense_hash_set::iterator it; - - for (it = set->begin(); it != set->end(); ++it) - { - abi_ulong elem; - elem = *it; - fwrite(&elem, sizeof(abi_ulong), 1, output_fp); - } -} - extern "C" void chatkey_setup_before_forkserver(void) { char * shm_id; @@ -68,16 +49,7 @@ extern "C" void chatkey_setup_after_forkserver(void) { char * dbg_path = getenv("CK_DBG_LOG"); char * coverage_path = getenv("CK_COVERAGE_LOG"); - if (chatkey_mode == -1) { - assert(getenv("CK_MODE") != NULL); - chatkey_mode = atoi(getenv("CK_MODE")); - } - //assert(getenv("CK_FORK_SERVER") != NULL); - //// If fork server is enabled, chatkey_mode should have been set already. - //if (atoi(getenv("CK_FORK_SERVER")) == 0) { - // assert(getenv("CK_MODE") != NULL); - // chatkey_mode = atoi(getenv("CK_MODE")); - //} + passed_forkserver = 1; /* Open file pointers and descriptors early, since if we try to open them in * chatkey_exit(), it gets mixed with stderr & stdout stream. This seems to @@ -98,20 +70,19 @@ extern "C" void chatkey_setup_after_forkserver(void) { // When fork() syscall is encountered, child process should call this function extern "C" void chatkey_close_fp(void) { - is_fp_closed = 1; - // close 'coverage_fp', since we don't want to dump log twice fclose(coverage_fp); + coverage_fp = NULL; if (afl_forksrv_pid) - close(TSL_FD); + close(TSL_FD); } extern "C" void chatkey_exit(void) { sigset_t mask; // If chatkey_close_fp() was called, then return without any action. - if (is_fp_closed) + if (coverage_fp == NULL) return; // Block signals, since we register signal handler that calls chatkey_exit()/ @@ -120,34 +91,17 @@ extern "C" void chatkey_exit(void) { if (sigprocmask(SIG_BLOCK, &mask, NULL) < 0) return; - if (chatkey_mode == MODE_COUNT_NEW) { - - /* Output new edge # and path hash. */ - fprintf(coverage_fp, "%d\n", new_edge_cnt); + /* Output new edge # and path hash. */ + fprintf(coverage_fp, "%d\n", new_edge_cnt); #ifdef TARGET_X86_64 - fprintf(coverage_fp, "%lu\n", path_hash); - fprintf(coverage_fp, "%lu\n", edge_set_hash); + fprintf(coverage_fp, "%lu\n", path_hash); + fprintf(coverage_fp, "%lu\n", edge_set_hash); #else - fprintf(coverage_fp, "%u\n", path_hash); - fprintf(coverage_fp, "%u\n", edge_set_hash); + fprintf(coverage_fp, "%u\n", path_hash); + fprintf(coverage_fp, "%u\n", edge_set_hash); #endif - fclose(coverage_fp); - } else if (chatkey_mode == MODE_HASH) { - /* Output path hash and edge hash */ -#ifdef TARGET_X86_64 - fprintf(coverage_fp, "%lu\n", edge_set_hash); -#else - fprintf(coverage_fp, "%u\n", edge_set_hash); -#endif - fclose(coverage_fp); - } else if (chatkey_mode == MODE_SET) { - /* Dump visited edge set */ - dump_set(&edge_set, coverage_fp); - fclose(coverage_fp); - } else { - assert(false); - } + fclose(coverage_fp); if (dbg_fp) fclose(dbg_fp); @@ -177,7 +131,7 @@ extern "C" void chatkey_log_bb(abi_ulong addr, abi_ulong callsite) { #endif prev_node = addr; - if (chatkey_mode == MODE_COUNT_NEW) { + if (passed_forkserver) { // Check and update both edge_bitmap and accumulative bitmap hash = (edge >> 4) ^ (edge << 8); byte_idx = (hash >> 3) & 0xffff; @@ -203,26 +157,5 @@ extern "C" void chatkey_log_bb(abi_ulong addr, abi_ulong callsite) { } } } - } else if (chatkey_mode == MODE_HASH) { - // Check and update edge_bitmap only - hash = (edge >> 4) ^ (edge << 8); - byte_idx = (hash >> 3) & 0xffff; - byte_mask = 1 << (hash & 0x7); // Lowest 3 bits - old_byte = edge_bitmap[byte_idx]; - new_byte = old_byte | byte_mask; - if (old_byte != new_byte) { - edge_bitmap[byte_idx] = new_byte; - // If it's a new edge, update edge hash and also add to accumulative map - update_edge_set_hash(hash); - } - } else if (chatkey_mode == MODE_SET ) { - // Just insert currently covered edge to the edge set - edge_set.insert(edge); - } else if (chatkey_mode != -1) { - /* If chatkey_mode is -1, it means that chatkey_setup() is not called yet - * This happens when QEMU is executing a dynamically linked program. Other - * values mean error. - */ - assert(false); } } diff --git a/Instrumentor/qemu/patches-pathcov/cpu-exec.diff b/Instrumentor/qemu/patches-coverage/cpu-exec.diff similarity index 71% rename from Instrumentor/qemu/patches-pathcov/cpu-exec.diff rename to Instrumentor/qemu/patches-coverage/cpu-exec.diff index da4a4c3..4005216 100644 --- a/Instrumentor/qemu/patches-pathcov/cpu-exec.diff +++ b/Instrumentor/qemu/patches-coverage/cpu-exec.diff @@ -1,6 +1,6 @@ ---- qemu-2.3.0-pathcov/cpu-exec.c.orig 2019-02-04 15:58:26.783108755 +0900 -+++ qemu-2.3.0-pathcov/cpu-exec.c 2019-02-04 16:01:19.912952716 +0900 -@@ -28,6 +28,14 @@ +--- qemu-2.3.0-coverage/cpu-exec.c.orig 2020-09-21 23:23:17.517246984 -0700 ++++ qemu-2.3.0-coverage/cpu-exec.c 2020-09-21 23:22:27.909433624 -0700 +@@ -28,6 +28,13 @@ #include "exec/memory-internal.h" #include "qemu/rcu.h" @@ -10,23 +10,11 @@ +extern void chatkey_log_bb(abi_ulong addr, abi_ulong callsite); + +#include "afl-qemu-cpu-inl.h" -+#include "chatkey-utils.h" + /* -icount align implementation. */ typedef struct SyncClocks { -@@ -36,6 +44,10 @@ - int64_t realtime_clock; - } SyncClocks; - -+/* Chatkey */ -+int is_function_entry = 0; -+abi_ulong current_function = 0; -+ - #if !defined(CONFIG_USER_ONLY) - /* Allow the guest to have a max 3ms advance. - * The difference between the 2 clocks could therefore -@@ -296,8 +308,11 @@ +@@ -296,8 +303,11 @@ } not_found: /* if no translated code available, then translate it now */ @@ -38,7 +26,7 @@ found: /* Move the last found TB to the head of the list */ if (likely(*ptb1)) { -@@ -492,6 +507,15 @@ +@@ -492,6 +502,15 @@ next_tb = 0; tcg_ctx.tb_ctx.tb_invalidated_flag = 0; } diff --git a/Instrumentor/qemu/patches-pathcov/exec-all.diff b/Instrumentor/qemu/patches-coverage/exec-all.diff similarity index 59% rename from Instrumentor/qemu/patches-pathcov/exec-all.diff rename to Instrumentor/qemu/patches-coverage/exec-all.diff index e7aa406..d96db38 100644 --- a/Instrumentor/qemu/patches-pathcov/exec-all.diff +++ b/Instrumentor/qemu/patches-coverage/exec-all.diff @@ -1,5 +1,5 @@ ---- qemu-2.3.0-pathcov/include/exec/exec-all.h.orig 2017-10-07 02:08:36.745270225 +0900 -+++ qemu-2.3.0-pathcov/include/exec/exec-all.h 2017-10-07 00:34:56.355819708 +0900 +--- qemu-2.3.0-coverage/include/exec/exec-all.h.orig 2017-10-07 02:08:36.745270225 +0900 ++++ qemu-2.3.0-coverage/include/exec/exec-all.h 2017-10-07 00:34:56.355819708 +0900 @@ -173,6 +173,9 @@ jmp_first */ struct TranslationBlock *jmp_next[2]; diff --git a/Instrumentor/qemu/patches-pathcov/makefile-target.diff b/Instrumentor/qemu/patches-coverage/makefile-target.diff similarity index 80% rename from Instrumentor/qemu/patches-pathcov/makefile-target.diff rename to Instrumentor/qemu/patches-coverage/makefile-target.diff index 435d4a6..c58722f 100644 --- a/Instrumentor/qemu/patches-pathcov/makefile-target.diff +++ b/Instrumentor/qemu/patches-coverage/makefile-target.diff @@ -1,5 +1,5 @@ ---- qemu-2.3.0-pathcov/Makefile.target 2015-04-27 23:08:23.000000000 +0900 -+++ qemu-2.3.0-pathcov-new/Makefile.target 2016-07-12 15:54:44.666065500 +0900 +--- qemu-2.3.0-coverage/Makefile.target 2015-04-27 23:08:23.000000000 +0900 ++++ qemu-2.3.0-coverage-new/Makefile.target 2016-07-12 15:54:44.666065500 +0900 @@ -11,7 +11,7 @@ endif QEMU_CFLAGS += -I.. -I$(SRC_PATH)/target-$(TARGET_BASE_ARCH) -DNEED_CPU_H diff --git a/Instrumentor/qemu/patches-feedback/syscall.diff b/Instrumentor/qemu/patches-coverage/syscall.diff similarity index 98% rename from Instrumentor/qemu/patches-feedback/syscall.diff rename to Instrumentor/qemu/patches-coverage/syscall.diff index 5d3579c..8740728 100644 --- a/Instrumentor/qemu/patches-feedback/syscall.diff +++ b/Instrumentor/qemu/patches-coverage/syscall.diff @@ -1,5 +1,5 @@ ---- qemu-2.3.0-feedback/linux-user/syscall.c.orig 2017-07-20 14:41:00.160771159 +0900 -+++ qemu-2.3.0-feedback/linux-user/syscall.c 2017-07-20 14:34:26.127522059 +0900 +--- qemu-2.3.0-coverage/linux-user/syscall.c.orig 2017-07-20 14:40:31.912793680 +0900 ++++ qemu-2.3.0-coverage/linux-user/syscall.c 2017-07-20 14:25:16.001550285 +0900 @@ -115,6 +115,33 @@ #include "qemu.h" diff --git a/Instrumentor/qemu/patches-pathcov/translate.diff b/Instrumentor/qemu/patches-coverage/translate.diff similarity index 97% rename from Instrumentor/qemu/patches-pathcov/translate.diff rename to Instrumentor/qemu/patches-coverage/translate.diff index a3212b0..3d5ffe2 100644 --- a/Instrumentor/qemu/patches-pathcov/translate.diff +++ b/Instrumentor/qemu/patches-coverage/translate.diff @@ -1,5 +1,5 @@ ---- qemu-2.3.0-pathcov/target-i386/translate.c.orig 2017-10-07 02:23:34.787285527 +0900 -+++ qemu-2.3.0-pathcov/target-i386/translate.c 2017-10-07 02:15:08.076137509 +0900 +--- qemu-2.3.0-coverage/target-i386/translate.c.orig 2017-10-07 02:23:34.787285527 +0900 ++++ qemu-2.3.0-coverage/target-i386/translate.c 2017-10-07 02:15:08.076137509 +0900 @@ -391,7 +391,7 @@ tcg_gen_addi_tl(cpu_A0, cpu_A0, val); } diff --git a/Instrumentor/qemu/patches-pathcov/chatkey-utils.h b/Instrumentor/qemu/patches-pathcov/chatkey-utils.h deleted file mode 100644 index 2533e1a..0000000 --- a/Instrumentor/qemu/patches-pathcov/chatkey-utils.h +++ /dev/null @@ -1,50 +0,0 @@ -/* List structure to handle call stack information */ -struct node { - abi_ulong data; - struct node * next; -}; - -void push(abi_ulong x); -abi_ulong pop(void); -abi_ulong fetch_context(int sensitivity); - -struct node * head = NULL; - -void push(abi_ulong x) { - struct node * new_node = (struct node *) malloc(sizeof(struct node)); - /* If so, we're all doomed, anyway */ - assert(new_node != NULL); - - new_node->data = x; - new_node->next = head; - head = new_node; -} - -abi_ulong pop(void) { - struct node * top_node; - abi_ulong top_data; - /* This cannot happen as long as we track call/ret correctly */ - assert(head != NULL); - top_node = head; - top_data = head->data; - - /* Update list*/ - head = head->next; - - /* free() and return */ - free(top_node); - return top_data; -} - -abi_ulong fetch_context(int sensitivity) { - int ctx = 0; - int i; - struct node * ptr = head; - - for(i = 0; i < sensitivity && ptr != NULL ; i++) { - ctx = (ctx << 8) ^ ptr->data; - ptr = ptr->next; - } - - return ctx; -} diff --git a/Instrumentor/qemu/patches-syscall/chatkey.c b/Instrumentor/qemu/patches-syscall/chatkey.c deleted file mode 100644 index 2295151..0000000 --- a/Instrumentor/qemu/patches-syscall/chatkey.c +++ /dev/null @@ -1,144 +0,0 @@ -#include -#include -#include -#include - -#include "qemu.h" - -#ifdef TARGET_X86_64 -typedef int64_t abi_long; -typedef uint64_t abi_ulong; - -#define TARGET_NR_open 2 -#define TARGET_NR_openat 257 -#define TARGET_NR_read 0 -#define TARGET_NR_dup 32 -#define TARGET_NR_dup2 33 -#define TARGET_NR_dup3 292 -#define TARGET_NR_exit_group 231 -#else -typedef int32_t abi_long; -typedef uint32_t abi_ulong; - -#define TARGET_NR_open 5 -#define TARGET_NR_openat 295 -#define TARGET_NR_read 3 -#define TARGET_NR_dup 41 -#define TARGET_NR_dup2 63 -#define TARGET_NR_dup3 330 -#define TARGET_NR_exit_group 252 -#endif - -#define BUFSIZE 4096 - -abi_ulong chatkey_entry_point; - -static FILE *log_fp = NULL; - -void chatkey_setup(void); -void chatkey_close_fp(void); -void chatkey_exit(void); -void chatkey_pre_syscall(int num, abi_long arg1, abi_long arg2); -void chatkey_post_syscall(int num, abi_long arg1, abi_long arg2, abi_long ret); -void escape_white_space(char * dst, char * src, int size); - -void escape_white_space(char * dst, char * src, int size) { - char * ptr_dst = dst; - char * ptr_src = src; - - while(*ptr_src && ptr_dst < dst + size - 4) { - if (*ptr_src == '\n') { - ptr_src++; - *ptr_dst++ = '\\'; - *ptr_dst++ = 'n'; - } else if (*ptr_src == '\r') { - ptr_src++; - *ptr_dst++ = '\\'; - *ptr_dst++ = 'r'; - } else if (*ptr_src == ' ') { - ptr_src++; - *ptr_dst++ = '\\'; - *ptr_dst++ = 's'; - } else if (*ptr_src == '\t') { - ptr_src++; - *ptr_dst++ = '\\'; - *ptr_dst++ = 't'; - } else { - *ptr_dst++ = *ptr_src++; - } - } - *ptr_dst = '\0'; -} - -void chatkey_setup(void) { - char *log_path = getenv("CK_SYSCALL_LOG"); - if (!log_path || log_fp) - return; - log_fp = fopen(log_path, "w"); -} - -// When fork() syscall is encountered, child process should call this function -void chatkey_close_fp(void) { - // close 'log_fp', since we don't want to dump log twice - fclose(log_fp); - log_fp = NULL; -} - -void chatkey_exit(void) { - sigset_t mask; - - // Block signals, since we register signal handler that calls chatkey_exit() - if (sigfillset(&mask) < 0) - return; - if (sigprocmask(SIG_BLOCK, &mask, NULL) < 0) - return; - - if (log_fp != NULL) - fclose(log_fp); -} - -void chatkey_pre_syscall(int num, abi_long arg1, abi_long arg2) { - if (!log_fp) - return; - - switch (num) { - case TARGET_NR_read: - fprintf(log_fp, "read %d\n", (int) arg1); - break; - case TARGET_NR_dup2: - case TARGET_NR_dup3: - fprintf(log_fp, "dup %d %d\n", (int) arg1, (int) arg2); - break; - default: - break; - } -} - -void chatkey_post_syscall(int num, abi_long arg1, abi_long arg2, abi_long ret) { - char filename[BUFSIZE]; - char* addr; - - /* If log_fp is NULL, it means that chatkey_setup() is not called yet. This - * happens when QEMU is executing a dynamically linked program. - */ - if (!log_fp) - return; - - switch (num) { - case TARGET_NR_open: - addr = (char*) ((abi_ulong) arg1 + guest_base); - escape_white_space(filename, addr, BUFSIZE); - fprintf(log_fp, "open %d %s\n", (int) ret, filename); - break; - case TARGET_NR_openat: - addr = (char*) ((abi_ulong) arg2 + guest_base); - escape_white_space(filename, addr, BUFSIZE); - fprintf(log_fp, "open %d %s\n", (int) ret, filename); - break; - case TARGET_NR_dup: - fprintf(log_fp, "dup %d %d\n", (int) arg1, (int) ret); - break; - default: - break; - } -} diff --git a/Instrumentor/qemu/patches-syscall/cpu-exec.diff b/Instrumentor/qemu/patches-syscall/cpu-exec.diff deleted file mode 100644 index b38a69c..0000000 --- a/Instrumentor/qemu/patches-syscall/cpu-exec.diff +++ /dev/null @@ -1,24 +0,0 @@ ---- qemu-2.3.0-syscall/cpu-exec.c.orig 2018-05-24 11:30:49.615769240 +0900 -+++ qemu-2.3.0-syscall/cpu-exec.c 2018-05-23 17:47:35.161933753 +0900 -@@ -28,6 +28,9 @@ - #include "exec/memory-internal.h" - #include "qemu/rcu.h" - -+extern abi_ulong chatkey_entry_point; -+extern void chatkey_setup(void); -+ - /* -icount align implementation. */ - - typedef struct SyncClocks { -@@ -492,6 +495,11 @@ - next_tb = 0; - tcg_ctx.tb_ctx.tb_invalidated_flag = 0; - } -+ -+ if(tb->pc == chatkey_entry_point) { -+ chatkey_setup(); -+ } -+ - if (qemu_loglevel_mask(CPU_LOG_EXEC)) { - qemu_log("Trace %p [" TARGET_FMT_lx "] %s\n", - tb->tc_ptr, tb->pc, lookup_symbol(tb->pc)); diff --git a/Instrumentor/qemu/patches-syscall/makefile-objs.diff b/Instrumentor/qemu/patches-syscall/makefile-objs.diff deleted file mode 100644 index 7ba5d55..0000000 --- a/Instrumentor/qemu/patches-syscall/makefile-objs.diff +++ /dev/null @@ -1,9 +0,0 @@ ---- qemu-2.3.0-syscall/linux-user/Makefile.objs.orig 2018-05-24 11:30:49.615769240 +0900 -+++ qemu-2.3.0-syscall/linux-user/Makefile.objs 2018-05-23 17:47:35.165933727 +0900 -@@ -1,5 +1,5 @@ - obj-y = main.o syscall.o strace.o mmap.o signal.o \ -- elfload.o linuxload.o uaccess.o uname.o -+ elfload.o linuxload.o uaccess.o uname.o chatkey.o - - obj-$(TARGET_HAS_BFLT) += flatload.o - obj-$(TARGET_I386) += vm86.o diff --git a/Instrumentor/qemu/patches-syscall/syscall.diff b/Instrumentor/qemu/patches-syscall/syscall.diff deleted file mode 100644 index be2f285..0000000 --- a/Instrumentor/qemu/patches-syscall/syscall.diff +++ /dev/null @@ -1,256 +0,0 @@ ---- qemu-2.3.0-syscall/linux-user/syscall.c.orig 2018-05-24 11:30:49.611769286 +0900 -+++ qemu-2.3.0-syscall/linux-user/syscall.c 2018-05-24 11:30:45.659813975 +0900 -@@ -115,6 +115,37 @@ - - #include "qemu.h" - -+extern void chatkey_pre_syscall(int num, abi_long arg1, abi_long arg2); -+extern void chatkey_post_syscall(int num, abi_long arg1, abi_long arg2, abi_long ret); -+extern void chatkey_exit(void); -+extern void chatkey_close_fp(void); -+ -+int check_executable(const char* filename); -+ -+int check_executable(const char* filename) { -+ int fd, size, tmp; -+ if (eaccess(filename, R_OK) == 0 && eaccess(filename, X_OK == 0)) { -+ fd = open(filename, O_RDONLY); -+ if (fd < 0) -+ return -1; -+ size = lseek(fd, 0, SEEK_END); -+ lseek(fd, 0, SEEK_SET); -+ if (size < 4) { -+ close(fd); -+ return -1; -+ } -+ if (read(fd, &tmp, 4) != 4) { -+ close(fd); -+ return -1; -+ } -+ close(fd); -+ if (tmp == 0x464c457f) -+ return 0; -+ } -+ -+ return -1; -+} -+ - #define CLONE_NPTL_FLAGS2 (CLONE_SETTLS | \ - CLONE_PARENT_SETTID | CLONE_CHILD_SETTID | CLONE_CHILD_CLEARTID) - -@@ -846,7 +877,7 @@ - { - abi_ulong target_rlim_swap; - rlim_t result; -- -+ - target_rlim_swap = tswapal(target_rlim); - if (target_rlim_swap == TARGET_RLIM_INFINITY) - return RLIM_INFINITY; -@@ -854,7 +885,7 @@ - result = target_rlim_swap; - if (target_rlim_swap != (rlim_t)result) - return RLIM_INFINITY; -- -+ - return result; - } - -@@ -862,13 +893,13 @@ - { - abi_ulong target_rlim_swap; - abi_ulong result; -- -+ - if (rlim == RLIM_INFINITY || rlim != (abi_long)rlim) - target_rlim_swap = TARGET_RLIM_INFINITY; - else - target_rlim_swap = rlim; - result = tswapal(target_rlim_swap); -- -+ - return result; - } - -@@ -1183,9 +1214,9 @@ - abi_ulong target_cmsg_addr; - struct target_cmsghdr *target_cmsg; - socklen_t space = 0; -- -+ - msg_controllen = tswapal(target_msgh->msg_controllen); -- if (msg_controllen < sizeof (struct target_cmsghdr)) -+ if (msg_controllen < sizeof (struct target_cmsghdr)) - goto the_end; - target_cmsg_addr = tswapal(target_msgh->msg_control); - target_cmsg = lock_user(VERIFY_READ, target_cmsg_addr, msg_controllen, 1); -@@ -1255,7 +1286,7 @@ - socklen_t space = 0; - - msg_controllen = tswapal(target_msgh->msg_controllen); -- if (msg_controllen < sizeof (struct target_cmsghdr)) -+ if (msg_controllen < sizeof (struct target_cmsghdr)) - goto the_end; - target_cmsg_addr = tswapal(target_msgh->msg_control); - target_cmsg = lock_user(VERIFY_WRITE, target_cmsg_addr, msg_controllen, 0); -@@ -4293,7 +4324,7 @@ - } - unlock_user_struct(target_ldt_info, ptr, 1); - -- if (ldt_info.entry_number < TARGET_GDT_ENTRY_TLS_MIN || -+ if (ldt_info.entry_number < TARGET_GDT_ENTRY_TLS_MIN || - ldt_info.entry_number > TARGET_GDT_ENTRY_TLS_MAX) - return -TARGET_EINVAL; - seg_32bit = ldt_info.flags & 1; -@@ -4371,7 +4402,7 @@ - lp = (uint32_t *)(gdt_table + idx); - entry_1 = tswap32(lp[0]); - entry_2 = tswap32(lp[1]); -- -+ - read_exec_only = ((entry_2 >> 9) & 1) ^ 1; - contents = (entry_2 >> 10) & 3; - seg_not_present = ((entry_2 >> 15) & 1) ^ 1; -@@ -4387,8 +4418,8 @@ - (read_exec_only << 3) | (limit_in_pages << 4) | - (seg_not_present << 5) | (useable << 6) | (lm << 7); - limit = (entry_1 & 0xffff) | (entry_2 & 0xf0000); -- base_addr = (entry_1 >> 16) | -- (entry_2 & 0xff000000) | -+ base_addr = (entry_1 >> 16) | -+ (entry_2 & 0xff000000) | - ((entry_2 & 0xff) << 16); - target_ldt_info->base_addr = tswapal(base_addr); - target_ldt_info->limit = tswap32(limit); -@@ -4572,6 +4603,7 @@ - ret = fork(); - if (ret == 0) { - /* Child Process. */ -+ chatkey_close_fp(); - rcu_after_fork(); - cpu_clone_regs(env, newsp); - fork_end(1); -@@ -5532,6 +5564,8 @@ - if(do_strace) - print_syscall(num, arg1, arg2, arg3, arg4, arg5, arg6); - -+ chatkey_pre_syscall(num, arg1, arg2); -+ - switch(num) { - case TARGET_NR_exit: - /* In old applications this may be used to implement _exit(2). -@@ -5561,6 +5595,7 @@ - #ifdef TARGET_GPROF - _mcleanup(); - #endif -+ chatkey_exit(); - gdb_exit(cpu_env, arg1); - _exit(arg1); - ret = 0; /* avoid warning */ -@@ -5749,10 +5784,21 @@ - } - if (!(p = lock_user_string(arg1))) - goto execve_efault; -- ret = get_errno(execve(p, argp, envp)); -- unlock_user(p, arg1, 0); -- -- goto execve_end; -+ /* If execve() call is expected to successfully load and execute the -+ * specified program, call chatkey_exit() and abort, since the newly -+ * executed program is not out target. Also, if the new program is -+ * loaded and executed, it often ignores SIGTERM signal sent from -+ * the driver, and Chatkey falls into infinite loop. */ -+ if (check_executable(p) != 0) { -+ unlock_user(p, arg1, 0); -+ goto execve_efault; -+ } -+ chatkey_exit(); -+ exit(0); -+ //ret = get_errno(execve(p, argp, envp)); -+ //unlock_user(p, arg1, 0); -+ // -+ //goto execve_end; - - execve_efault: - ret = -TARGET_EFAULT; -@@ -7422,6 +7468,7 @@ - #ifdef TARGET_GPROF - _mcleanup(); - #endif -+ chatkey_exit(); - gdb_exit(cpu_env, arg1); - ret = get_errno(exit_group(arg1)); - break; -@@ -8316,7 +8363,7 @@ - break; - #if defined(TARGET_NR_fchownat) - case TARGET_NR_fchownat: -- if (!(p = lock_user_string(arg2))) -+ if (!(p = lock_user_string(arg2))) - goto efault; - ret = get_errno(fchownat(arg1, p, low2highuid(arg3), - low2highgid(arg4), arg5)); -@@ -8801,7 +8848,7 @@ - case TARGET_F_GETLK64: - #ifdef TARGET_ARM - if (((CPUARMState *)cpu_env)->eabi) { -- if (!lock_user_struct(VERIFY_READ, target_efl, arg3, 1)) -+ if (!lock_user_struct(VERIFY_READ, target_efl, arg3, 1)) - goto efault; - fl.l_type = tswap16(target_efl->l_type); - fl.l_whence = tswap16(target_efl->l_whence); -@@ -8812,7 +8859,7 @@ - } else - #endif - { -- if (!lock_user_struct(VERIFY_READ, target_fl, arg3, 1)) -+ if (!lock_user_struct(VERIFY_READ, target_fl, arg3, 1)) - goto efault; - fl.l_type = tswap16(target_fl->l_type); - fl.l_whence = tswap16(target_fl->l_whence); -@@ -8825,7 +8872,7 @@ - if (ret == 0) { - #ifdef TARGET_ARM - if (((CPUARMState *)cpu_env)->eabi) { -- if (!lock_user_struct(VERIFY_WRITE, target_efl, arg3, 0)) -+ if (!lock_user_struct(VERIFY_WRITE, target_efl, arg3, 0)) - goto efault; - target_efl->l_type = tswap16(fl.l_type); - target_efl->l_whence = tswap16(fl.l_whence); -@@ -8836,7 +8883,7 @@ - } else - #endif - { -- if (!lock_user_struct(VERIFY_WRITE, target_fl, arg3, 0)) -+ if (!lock_user_struct(VERIFY_WRITE, target_fl, arg3, 0)) - goto efault; - target_fl->l_type = tswap16(fl.l_type); - target_fl->l_whence = tswap16(fl.l_whence); -@@ -8852,7 +8899,7 @@ - case TARGET_F_SETLKW64: - #ifdef TARGET_ARM - if (((CPUARMState *)cpu_env)->eabi) { -- if (!lock_user_struct(VERIFY_READ, target_efl, arg3, 1)) -+ if (!lock_user_struct(VERIFY_READ, target_efl, arg3, 1)) - goto efault; - fl.l_type = tswap16(target_efl->l_type); - fl.l_whence = tswap16(target_efl->l_whence); -@@ -8863,7 +8910,7 @@ - } else - #endif - { -- if (!lock_user_struct(VERIFY_READ, target_fl, arg3, 1)) -+ if (!lock_user_struct(VERIFY_READ, target_fl, arg3, 1)) - goto efault; - fl.l_type = tswap16(target_fl->l_type); - fl.l_whence = tswap16(target_fl->l_whence); -@@ -9819,6 +9866,9 @@ - ret = -TARGET_ENOSYS; - break; - } -+ -+ chatkey_post_syscall(num, arg1, arg2, ret); -+ - fail: - #ifdef DEBUG - gemu_log(" = " TARGET_ABI_FMT_ld "\n", ret); diff --git a/Instrumentor/qemu/prepare_qemu.sh b/Instrumentor/qemu/prepare_qemu.sh index d2799af..ab07b61 100755 --- a/Instrumentor/qemu/prepare_qemu.sh +++ b/Instrumentor/qemu/prepare_qemu.sh @@ -33,7 +33,7 @@ if [ ! "`uname -s`" = "Linux" ]; then fi -if [ ! -f "patches-pathcov/chatkey.cc" -o ! -f "patches-syscall/chatkey.c" -o ! -f "patches-feedback/chatkey.c" ]; then +if [ ! -f "patches-coverage/chatkey.cc" -o ! -f "patches-branch/chatkey.c" ]; then echo "[-] Error: key files not found - wrong working directory?" exit 1 @@ -90,16 +90,13 @@ fi echo "[*] Uncompressing archive (this will take a while)..." rm -rf "qemu-2.3.0" || exit 1 -rm -rf "qemu-2.3.0-pathcov" || exit 1 -rm -rf "qemu-2.3.0-syscall" || exit 1 -rm -rf "qemu-2.3.0-feedback" || exit 1 +rm -rf "qemu-2.3.0-coverage" || exit 1 +rm -rf "qemu-2.3.0-branch" || exit 1 rm -rf "qemu-2.3.0-bbcount" || exit 1 -rm -rf "qemu-2.3.0-pathcov-x86" || exit 1 -rm -rf "qemu-2.3.0-pathcov-x64" || exit 1 -rm -rf "qemu-2.3.0-syscall-x86" || exit 1 -rm -rf "qemu-2.3.0-syscall-x64" || exit 1 -rm -rf "qemu-2.3.0-feedback-x86" || exit 1 -rm -rf "qemu-2.3.0-feedback-x64" || exit 1 +rm -rf "qemu-2.3.0-coverage-x86" || exit 1 +rm -rf "qemu-2.3.0-coverage-x64" || exit 1 +rm -rf "qemu-2.3.0-branch-x86" || exit 1 +rm -rf "qemu-2.3.0-branch-x64" || exit 1 rm -rf "qemu-2.3.0-bbcount-x86" || exit 1 rm -rf "qemu-2.3.0-bbcount-x64" || exit 1 tar xf "$ARCHIVE" || exit 1 @@ -126,52 +123,40 @@ patch -p0 ] extern void set_env (string env_variable, string env_value) [] extern void initialize_exec (TimeoutHandling is_replay) [] extern int init_forkserver_coverage (int argc, string[] argv, uint64 timeout) @@ -46,10 +37,10 @@ type CoverageTracerMode = let buildDir = let exePath = System.Reflection.Assembly.GetEntryAssembly().Location System.IO.Path.GetDirectoryName(exePath) -let coverageTracerX86 = sprintf "%s/qemu-trace-pathcov-x86" buildDir -let coverageTracerX64 = sprintf "%s/qemu-trace-pathcov-x64" buildDir -let branchTracerX86 = sprintf "%s/qemu-trace-feedback-x86" buildDir -let branchTracerX64 = sprintf "%s/qemu-trace-feedback-x64" buildDir +let coverageTracerX86 = sprintf "%s/qemu-trace-coverage-x86" buildDir +let coverageTracerX64 = sprintf "%s/qemu-trace-coverage-x64" buildDir +let branchTracerX86 = sprintf "%s/qemu-trace-branch-x86" buildDir +let branchTracerX64 = sprintf "%s/qemu-trace-branch-x64" buildDir let bbCountTracerX86 = sprintf "%s/qemu-trace-bbcount-x86" buildDir let bbCountTracerX64 = sprintf "%s/qemu-trace-bbcount-x64" buildDir @@ -82,7 +73,6 @@ let initialize opt = if verbosity >= 2 then set_env("CK_DBG_LOG", System.IO.Path.GetFullPath(dbgLog)) // Set other environment variables in advance, to avoid affecting path hash. - set_env("CK_MODE", "0") set_env("CK_FEED_ADDR", "0") set_env("CK_FEED_IDX", "0") set_env("CK_CTX_SENSITIVITY", string CtxSensitivity) @@ -243,7 +233,6 @@ let private runBranchTracerForked opt (stdin: byte array) = (*** Top-level tracer executor functions ***) let getCoverage opt seed = - set_env("CK_MODE", string (int CoverageTracerMode.CountNewEdge)) setupFile seed let stdin = prepareStdIn seed let exitSig = if forkServerEnabled diff --git a/src/Core/libexec.c b/src/Core/libexec.c index 9c0c25f..0876791 100644 --- a/src/Core/libexec.c +++ b/src/Core/libexec.c @@ -398,12 +398,12 @@ void kill_forkserver() { int exec_fork_coverage(uint64_t timeout, int stdin_size, char *stdin_data) { int res, childstatus; static struct itimerval it; + static unsigned char tmp[4]; - int run_mode = atoi(getenv("CK_MODE")); - + /* TODO : what if we want to use pseudo-terminal? */ write_stdin(path_stdin_fd, stdin_size, stdin_data); - if ((res = write(path_fsrv_ctl_fd, &run_mode, 4)) != 4) { + if ((res = write(path_fsrv_ctl_fd, tmp, 4)) != 4) { perror("exec_fork_coverage: Cannot request new process to fork server"); printf("write() call ret = %d\n", res); return -1; @@ -455,11 +455,11 @@ int exec_fork_branch(uint64_t timeout, int stdin_size, char *stdin_data) { int res, childstatus; static struct itimerval it; - targ_addr = strtol(getenv("CK_FEED_ADDR"), NULL, 16); - targ_index = strtol(getenv("CK_FEED_IDX"), NULL, 16); /* TODO : what if we want to use pseudo-terminal? */ write_stdin(feed_stdin_fd, stdin_size, stdin_data); + targ_addr = strtol(getenv("CK_FEED_ADDR"), NULL, 16); + targ_index = strtol(getenv("CK_FEED_IDX"), NULL, 16); if ((res = write(feed_fsrv_ctl_fd, &targ_addr, 8)) != 8) { perror("exec_fork_branch: Cannot request new process to fork server (1)"); printf("write() call ret = %d\n", res); From bf886e01c25e545cd9f80426086edc0837658936 Mon Sep 17 00:00:00 2001 From: Jaeseung Choi Date: Tue, 22 Sep 2020 02:02:03 -0700 Subject: [PATCH 06/29] Polish example testing scripts --- examples/test_monoton_clang.sh | 3 ++- examples/test_monoton_gcc.sh | 3 ++- examples/test_motiv.sh | 2 +- 3 files changed, 5 insertions(+), 3 deletions(-) diff --git a/examples/test_monoton_clang.sh b/examples/test_monoton_clang.sh index ff66073..5b90e30 100755 --- a/examples/test_monoton_clang.sh +++ b/examples/test_monoton_clang.sh @@ -1,6 +1,7 @@ #!/bin/bash -# Grey-box concolic should find test cases that have \x41\x42 and "Good!". +# Grey-box concolic should find test cases that have \x41\x42, \x61\x62, +# "Good!", and "Bad!". clang monoton.c -o monoton.bin -static -g || exit 1 rm -rf box mkdir box diff --git a/examples/test_monoton_gcc.sh b/examples/test_monoton_gcc.sh index a840dac..b91ceb3 100755 --- a/examples/test_monoton_gcc.sh +++ b/examples/test_monoton_gcc.sh @@ -1,6 +1,7 @@ #!/bin/bash -# Grey-box concolic should find test cases that have \x41\x42 and "Good!". +# Grey-box concolic should find test cases that have \x41\x42, \x61\x62, +# "Good!", and "Bad!". gcc monoton.c -o monoton.bin -static -g || exit 1 rm -rf box mkdir box diff --git a/examples/test_motiv.sh b/examples/test_motiv.sh index ecb093d..32d831c 100755 --- a/examples/test_motiv.sh +++ b/examples/test_motiv.sh @@ -1,4 +1,4 @@ -!/bin/bash +#!/bin/bash # Grey-box concolic should find a program-crashing test case within few seconds. gcc motiv.c -o motiv.bin -static -g || exit 1 From d15234273ef9462d56d50fd61f1e2cdc98ce8615 Mon Sep 17 00:00:00 2001 From: Jaeseung Choi Date: Wed, 23 Sep 2020 02:16:42 -0700 Subject: [PATCH 07/29] Fix QEMU instrumentation design - Make branch tracer to measure coverage along with branch trace. - Use memory-mapped file instead of shared memory. - Remove unused variables and functions. --- .../qemu/patches-branch/afl-qemu-cpu-inl.h | 17 +- Instrumentor/qemu/patches-branch/chatkey.c | 157 +++++++++------- .../qemu/patches-branch/cpu-exec.diff | 56 ++---- .../qemu/patches-branch/translate.diff | 2 +- .../qemu/patches-coverage/afl-qemu-cpu-inl.h | 5 +- Instrumentor/qemu/patches-coverage/chatkey.cc | 147 ++++++--------- .../qemu/patches-coverage/cpu-exec.diff | 4 +- Instrumentor/qemu/repatch.sh | 2 - src/Core/Config.fs | 11 +- src/Core/Executor.fs | 171 +++++++----------- src/Core/Options.fs | 2 +- src/Core/Seed.fs | 14 +- src/Core/Typedef.fs | 6 + src/Core/Utils.fs | 21 +-- src/Core/libexec.c | 153 ++++++---------- src/Eclipser.fsproj | 2 +- src/Fuzz/Fuzz.fs | 44 +++-- src/Fuzz/Manager.fs | 33 +--- src/GreyConcolic/BranchTrace.fs | 20 +- src/GreyConcolic/BranchTree.fs | 4 +- src/GreyConcolic/GreyConcolic.fs | 4 +- src/GreyConcolic/Solve.fs | 95 +++++----- 22 files changed, 401 insertions(+), 569 deletions(-) mode change 100644 => 100755 src/Core/libexec.c diff --git a/Instrumentor/qemu/patches-branch/afl-qemu-cpu-inl.h b/Instrumentor/qemu/patches-branch/afl-qemu-cpu-inl.h index 9d280e1..3bbfe48 100644 --- a/Instrumentor/qemu/patches-branch/afl-qemu-cpu-inl.h +++ b/Instrumentor/qemu/patches-branch/afl-qemu-cpu-inl.h @@ -24,8 +24,8 @@ #define TSL_FD (FORKSRV_FD - 1) extern abi_ulong chatkey_targ_addr; -extern size_t chatkey_targ_index; -extern abi_ulong mmap_next_start; +extern uint32_t chatkey_targ_index; +extern int measure_coverage; /* Set in the child process in forkserver mode: */ @@ -58,12 +58,10 @@ struct afl_tsl { /* Fork server logic, invoked once we hit _start. */ static void afl_forkserver(CPUArchState *env) { - - uint64_t tmp_input; static unsigned char tmp[4]; + uint64_t tmp_input; - if (!getenv("CK_FEED_LOG")) - return; + if (atoi(getenv("CK_FORK_SERVER")) != 1) return; /* Tell the parent that we're alive. If the parent doesn't want to talk, assume that we're not running in forkserver mode. */ @@ -83,8 +81,8 @@ static void afl_forkserver(CPUArchState *env) { if (read(FORKSRV_FD, &tmp_input, 8) != 8) exit(2); chatkey_targ_addr = (abi_ulong)tmp_input; - if (read(FORKSRV_FD, &tmp_input, 8) != 8) exit(2); - chatkey_targ_index = (size_t)tmp_input; + if (read(FORKSRV_FD, &chatkey_targ_index, 4) != 4) exit(2); + if (read(FORKSRV_FD, &measure_coverage, 4) != 4) exit(2); /* Establish a channel with child to grab translation commands. We'll read from t_fd[0], child will write to TSL_FD. */ @@ -161,8 +159,7 @@ static void afl_wait_tsl(CPUArchState *env, int fd) { if (read(fd, &t, sizeof(struct afl_tsl)) != sizeof(struct afl_tsl)) break; - if (t.pc < mmap_next_start) - tb_find_slow(env, t.pc, t.cs_base, t.flags); + tb_find_slow(env, t.pc, t.cs_base, t.flags); } diff --git a/Instrumentor/qemu/patches-branch/chatkey.c b/Instrumentor/qemu/patches-branch/chatkey.c index 07e9451..a35775a 100644 --- a/Instrumentor/qemu/patches-branch/chatkey.c +++ b/Instrumentor/qemu/patches-branch/chatkey.c @@ -1,11 +1,11 @@ #include -#include #include #include #include #include #include - +#include +#include #include "tcg.h" #ifdef TARGET_X86_64 @@ -20,64 +20,76 @@ extern unsigned int afl_forksrv_pid; #define MAX_TRACE_LEN (1000000) -static abi_ulong hash = 5381; // djb2 hash +abi_ulong chatkey_entry_point = 0; /* ELF entry point (_start) */ +abi_ulong chatkey_curr_addr = 0; +abi_ulong chatkey_targ_addr = 0; +uint32_t chatkey_targ_index = 0; +int measure_coverage = 0; +int chatkey_EP_passed = 0; + +static int found_new_edge = 0; +static int found_new_path = 0; // TODO. Extend to measure path coverage, too. +static abi_ulong prev_node = 0; +static char * coverage_path = NULL; +static char * branch_path = NULL; +static FILE * coverage_fp = NULL; +static FILE * branch_fp = NULL; +static unsigned char * accum_edge_bitmap = NULL; -abi_ulong chatkey_entry_point; -abi_ulong chatkey_curr_addr; -abi_ulong chatkey_targ_addr; unsigned char trace_buffer[MAX_TRACE_LEN * (sizeof(abi_ulong) + sizeof(unsigned char) + 2 * sizeof(abi_ulong)) + 64]; unsigned char * buf_ptr = trace_buffer; -size_t chatkey_targ_index; -bool chatkey_EP_passed = false; - -static size_t targ_hit_count = 0; -static size_t trace_count = 0; -static FILE * branch_fp; -static FILE * hash_fp; - -void flush_trace_buffer(void); -void chatkey_setup(void); -void chatkey_close_fp(void); -void chatkey_exit(void); -void chatkey_log_branch(abi_ulong oprnd1, abi_ulong oprnd2, unsigned char operand_type); -void chatkey_update_hash(register abi_ulong addr); +static uint32_t targ_hit_count = 0; +static uint32_t trace_count = 0; void flush_trace_buffer(void) { size_t len = buf_ptr - trace_buffer; fwrite(trace_buffer, len, 1, branch_fp); } -void chatkey_setup(void) { +void chatkey_setup_before_forkserver(void) { + char * bitmap_path = getenv("CK_BITMAP_LOG"); + int bitmap_fd = open(bitmap_path, O_RDWR | O_CREAT, 0644); + accum_edge_bitmap = (unsigned char*) mmap(NULL, 0x10000, PROT_READ | PROT_WRITE, MAP_SHARED, bitmap_fd, 0); + assert(accum_edge_bitmap != (void *) -1); - char * branch_path = getenv("CK_FEED_LOG"); - char * hash_path = getenv("CK_HASH_LOG"); + coverage_path = getenv("CK_COVERAGE_LOG"); + branch_path = getenv("CK_FEED_LOG"); - assert(branch_path != NULL); - branch_fp = fopen(branch_path, "w"); - assert(branch_fp != NULL); + chatkey_EP_passed = 1; +} - assert(hash_path != NULL); - hash_fp = fopen(hash_path, "w"); - assert(hash_fp != NULL); +void chatkey_setup_after_forkserver(void) { assert(getenv("CK_FORK_SERVER") != NULL); - // If fork server is enabled, chatkey_targ_* should have been set already. + // If fork server is enabled, the following data are set during the handshake. if (atoi(getenv("CK_FORK_SERVER")) == 0) { chatkey_targ_addr = strtol(getenv("CK_FEED_ADDR"), NULL, 16); chatkey_targ_index = strtol(getenv("CK_FEED_IDX"), NULL, 16); + measure_coverage = atoi(getenv("CK_MEASURE_COV")); + } + + if (measure_coverage == 1) { + coverage_fp = fopen(coverage_path, "w"); + assert(coverage_fp != NULL); } + branch_fp = fopen(branch_path, "w"); + assert(branch_fp != NULL); } -// When fork() syscall is encountered, child process should call this function +// When fork() syscall is encountered, child process should call this function. void chatkey_close_fp(void) { + // Close file pointers, to avoid dumping log twice. + if (coverage_fp) { + fclose(coverage_fp); + coverage_fp = NULL; + } - // close 'branch_fp', since we don't want to dump log twice - fclose(branch_fp); - branch_fp = NULL; - - fclose(hash_fp); + if (branch_fp) { + fclose(branch_fp); + branch_fp = NULL; + } if (afl_forksrv_pid) close(TSL_FD); @@ -87,27 +99,24 @@ void chatkey_exit(void) { abi_ulong nil = 0; sigset_t mask; - // If chatkey_close_fp() was called, then return without any action - if (branch_fp == NULL) - return; - // Block signals, since we register signal handler that calls chatkey_exit() if (sigfillset(&mask) < 0) return; if (sigprocmask(SIG_BLOCK, &mask, NULL) < 0) return; - flush_trace_buffer(); - - fwrite(&nil, sizeof(abi_ulong), 1, branch_fp); - fclose(branch_fp); + if (coverage_fp) { + fprintf(coverage_fp, "%d\n%d\n", found_new_edge, found_new_path); + fclose(coverage_fp); + coverage_fp = NULL; + } -#ifdef TARGET_X86_64 - fprintf(hash_fp, "%lu\n", hash); -#else - fprintf(hash_fp, "%u\n", hash); -#endif - fclose(hash_fp); + if (branch_fp) { + flush_trace_buffer(); + fwrite(&nil, sizeof(abi_ulong), 1, branch_fp); + fclose(branch_fp); + branch_fp = NULL; + } } /* Recall that in 64bit we already pushed rdi/rsi/rdx before calling @@ -182,15 +191,17 @@ void chatkey_log_branch(abi_ulong oprnd1, abi_ulong oprnd2, unsigned char type) fwrite(&type, sizeof(unsigned char), 1, branch_fp); fwrite(&oprnd1_truncated, operand_size, 1, branch_fp); fwrite(&oprnd2_truncated, operand_size, 1, branch_fp); - if (oprnd1_truncated != oprnd2_truncated) { - /* If two operands are not equal, then path hash of this execution is - * not used in Chatkey. Therefore, finish execution to save time. + if (oprnd1_truncated != oprnd2_truncated || !coverage_fp) { + /* If the two operands are not equal, exit signal or coverage gain is + * not used in F# code. Simiarly, when coverage_fp is NULL, this means + * we are interested in branch distance only, and not in exit signal or + * coverage gain. In these case, halt the execution here to save time. */ chatkey_exit(); exit(0); } } - } else if (trace_count ++ < MAX_TRACE_LEN) { + } else if (trace_count++ < MAX_TRACE_LEN) { /* We're in the mode that traces all the cmp/test instructions */ if (operand_type == MO_8) { oprnd1_truncated = oprnd1 & 0xff; @@ -241,20 +252,36 @@ void chatkey_log_branch(abi_ulong oprnd1, abi_ulong oprnd2, unsigned char type) } else { /* We're in the mode that traces all the cmp/test instructions, and trace * limit has exceeded. Abort tracing. */ - abi_ulong nil = 0; - flush_trace_buffer(); - fwrite(&nil, sizeof(abi_ulong), 1, branch_fp); - fclose(branch_fp); - // output 0 as path hash to indicate abortion. - fprintf(hash_fp, "0\n"); - fclose(hash_fp); + chatkey_exit(); exit(0); } } +void chatkey_log_bb(abi_ulong addr) { + abi_ulong edge, hash; + unsigned int byte_idx, byte_mask; + unsigned char old_byte, new_byte; + + chatkey_curr_addr = addr; + + if (!coverage_fp) + return; -void chatkey_update_hash(register abi_ulong addr) { - register unsigned int i; - for (i = 0; i < sizeof(abi_ulong); i++) - hash = ((hash << 5) + hash) + ((addr >> (i<<3)) & 0xff); +#ifdef TARGET_X86_64 + edge = (prev_node << 16) ^ addr; +#else + edge = (prev_node << 8) ^ addr; +#endif + prev_node = addr; + + // Update bitmap. + hash = (edge >> 4) ^ (edge << 8); + byte_idx = (hash >> 3) & 0xffff; + byte_mask = 1 << (hash & 0x7); // Use the lowest 3 bits to shift + old_byte = accum_edge_bitmap[byte_idx]; + new_byte = old_byte | byte_mask; + if (old_byte != new_byte) { + found_new_edge = 1; + accum_edge_bitmap[byte_idx] = new_byte; + } } diff --git a/Instrumentor/qemu/patches-branch/cpu-exec.diff b/Instrumentor/qemu/patches-branch/cpu-exec.diff index 3186a77..56dd025 100644 --- a/Instrumentor/qemu/patches-branch/cpu-exec.diff +++ b/Instrumentor/qemu/patches-branch/cpu-exec.diff @@ -1,22 +1,24 @@ --- qemu-2.3.0-branch/cpu-exec.c.orig 2017-09-17 01:29:39.966426739 +0900 +++ qemu-2.3.0-branch/cpu-exec.c 2017-09-17 01:28:53.506606640 +0900 -@@ -28,6 +28,14 @@ +@@ -28,6 +28,13 @@ #include "exec/memory-internal.h" #include "qemu/rcu.h" -+extern abi_ulong chatkey_entry_point; -+extern abi_ulong chatkey_curr_addr; -+extern bool chatkey_EP_passed; -+extern void chatkey_setup(void); -+extern void chatkey_update_hash(abi_ulong addr); ++extern abi_ulong chatkey_entry_point; /* ELF entry point (_start) */ ++extern void chatkey_setup_before_forkserver(void); ++extern void chatkey_setup_after_forkserver(void); ++extern void chatkey_log_bb(abi_ulong addr); + +#include "afl-qemu-cpu-inl.h" + /* -icount align implementation. */ typedef struct SyncClocks { -@@ -298,6 +306,8 @@ +@@ -296,8 +303,11 @@ + } + not_found: /* if no translated code available, then translate it now */ ++ tb = tb_gen_code(cpu, pc, cs_base, flags, 0); + afl_request_tsl(pc, cs_base, flags); @@ -24,51 +26,19 @@ found: /* Move the last found TB to the head of the list */ if (likely(*ptb1)) { -@@ -480,6 +490,7 @@ - cpu->exception_index = EXCP_INTERRUPT; - cpu_loop_exit(cpu); - } -+ - spin_lock(&tcg_ctx.tb_ctx.tb_lock); - have_tb_lock = true; - tb = tb_find_fast(env); -@@ -492,6 +503,22 @@ +@@ -492,6 +502,15 @@ next_tb = 0; tcg_ctx.tb_ctx.tb_invalidated_flag = 0; } + + if(tb->pc == chatkey_entry_point) { -+ /* XXX. If the following line is moved into chatkey_setup(), a -+ * weired bug occurs when fork server is enabled. In the first -+ * execution, execution branchs are correcly collected. But -+ * in the subsequent executions, chatkey_log_branch() is not -+ * called at all. -+ */ -+ chatkey_EP_passed = true; ++ chatkey_setup_before_forkserver(); + afl_forkserver(env); -+ chatkey_setup(); ++ chatkey_setup_after_forkserver(); + } + -+ chatkey_curr_addr = tb->pc; -+ chatkey_update_hash(tb->pc); ++ chatkey_log_bb(tb->pc); + if (qemu_loglevel_mask(CPU_LOG_EXEC)) { qemu_log("Trace %p [" TARGET_FMT_lx "] %s\n", tb->tc_ptr, tb->pc, lookup_symbol(tb->pc)); -@@ -499,10 +526,12 @@ - /* see if we can patch the calling TB. When the TB - spans two pages, we cannot safely do a direct - jump. */ -- if (next_tb != 0 && tb->page_addr[1] == -1) { -- tb_add_jump((TranslationBlock *)(next_tb & ~TB_EXIT_MASK), -- next_tb & TB_EXIT_MASK, tb); -- } -+ /* XXX: chatkey. Commented out by tunz. -+ * if (next_tb != 0 && tb->page_addr[1] == -1) { -+ * tb_add_jump((TranslationBlock *)(next_tb & ~TB_EXIT_MASK), -+ * next_tb & TB_EXIT_MASK, tb); -+ *} -+ */ - have_tb_lock = false; - spin_unlock(&tcg_ctx.tb_ctx.tb_lock); - diff --git a/Instrumentor/qemu/patches-branch/translate.diff b/Instrumentor/qemu/patches-branch/translate.diff index d5806ab..9110c50 100644 --- a/Instrumentor/qemu/patches-branch/translate.diff +++ b/Instrumentor/qemu/patches-branch/translate.diff @@ -7,7 +7,7 @@ +/* A variable introduced in chatkey.c, to decide whether we should start + * instrumentation. + */ -+extern bool chatkey_EP_passed; ++extern int chatkey_EP_passed; +extern abi_ulong chatkey_curr_addr; +extern abi_ulong chatkey_targ_addr; + diff --git a/Instrumentor/qemu/patches-coverage/afl-qemu-cpu-inl.h b/Instrumentor/qemu/patches-coverage/afl-qemu-cpu-inl.h index 47ba5ae..6185c7a 100644 --- a/Instrumentor/qemu/patches-coverage/afl-qemu-cpu-inl.h +++ b/Instrumentor/qemu/patches-coverage/afl-qemu-cpu-inl.h @@ -23,8 +23,6 @@ #define FORKSRV_FD 198 #define TSL_FD (FORKSRV_FD - 1) -extern abi_ulong mmap_next_start; - /* Set in the child process in forkserver mode: */ static unsigned char afl_fork_child; @@ -59,8 +57,7 @@ static void afl_forkserver(CPUArchState *env) { static unsigned char tmp[4]; - if (!getenv("CK_COVERAGE_LOG")) - return; + if (atoi(getenv("CK_FORK_SERVER")) != 1) return; /* Tell the parent that we're alive. If the parent doesn't want to talk, assume that we're not running in forkserver mode. */ diff --git a/Instrumentor/qemu/patches-coverage/chatkey.cc b/Instrumentor/qemu/patches-coverage/chatkey.cc index 3fc5f43..02c91bb 100644 --- a/Instrumentor/qemu/patches-coverage/chatkey.cc +++ b/Instrumentor/qemu/patches-coverage/chatkey.cc @@ -1,13 +1,11 @@ #include -#include -#include -#include +#include +#include #include #include #include -#include - -using google::dense_hash_set; +#include +#include #ifdef TARGET_X86_64 typedef uint64_t abi_ulong; @@ -21,42 +19,31 @@ extern unsigned int afl_forksrv_pid; abi_ulong chatkey_entry_point; /* ELF entry point (_start) */ -int passed_forkserver = 0; -static uint32_t new_edge_cnt = 0; // # of new edges visited in this execution -static abi_ulong edge_set_hash = 5381; // djb2 hash -static abi_ulong path_hash = 5381; // djb2 hash +static int found_new_edge = 0; +static int found_new_path = 0; // TODO. Extend to measure path coverage, too. static abi_ulong prev_node = 0; -static FILE* coverage_fp; -static FILE* dbg_fp; - +static char * coverage_path = NULL; +static char * dbg_path = NULL; +static FILE * coverage_fp = NULL; +static FILE * dbg_fp = NULL; static unsigned char * accum_edge_bitmap; -static unsigned char edge_bitmap[0x10000]; -// Holds edges visited in this exec (will be dumped into a file) -static dense_hash_set edge_set; extern "C" void chatkey_setup_before_forkserver(void) { - char * shm_id; - - shm_id = getenv("CK_SHM_ID"); - assert(shm_id != NULL); - accum_edge_bitmap = (unsigned char *) shmat(atoi(shm_id), NULL, 0); + char * bitmap_path = getenv("CK_BITMAP_LOG"); + int bitmap_fd = open(bitmap_path, O_RDWR | O_CREAT, 0644); + accum_edge_bitmap = (unsigned char*) mmap(NULL, 0x10000, PROT_READ | PROT_WRITE, MAP_SHARED, bitmap_fd, 0); assert(accum_edge_bitmap != (void *) -1); - edge_set.set_empty_key(0); + coverage_path = getenv("CK_COVERAGE_LOG"); + dbg_path = getenv("CK_DBG_LOG"); } extern "C" void chatkey_setup_after_forkserver(void) { - char * dbg_path = getenv("CK_DBG_LOG"); - char * coverage_path = getenv("CK_COVERAGE_LOG"); - - passed_forkserver = 1; - /* Open file pointers and descriptors early, since if we try to open them in * chatkey_exit(), it gets mixed with stderr & stdout stream. This seems to * be an issue due to incorrect file descriptor management in QEMU code. */ - assert(coverage_path != NULL); coverage_fp = fopen(coverage_path, "w"); assert(coverage_fp != NULL); @@ -67,12 +54,18 @@ extern "C" void chatkey_setup_after_forkserver(void) { } } -// When fork() syscall is encountered, child process should call this function +// When fork() syscall is encountered, child process should call this function. extern "C" void chatkey_close_fp(void) { + // Close file pointers, to avoid dumping log twice. + if (coverage_fp) { + fclose(coverage_fp); + coverage_fp = NULL; + } - // close 'coverage_fp', since we don't want to dump log twice - fclose(coverage_fp); - coverage_fp = NULL; + if (dbg_fp) { + fclose(dbg_fp); + dbg_fp = NULL; + } if (afl_forksrv_pid) close(TSL_FD); @@ -81,81 +74,55 @@ extern "C" void chatkey_close_fp(void) { extern "C" void chatkey_exit(void) { sigset_t mask; - // If chatkey_close_fp() was called, then return without any action. - if (coverage_fp == NULL) - return; - // Block signals, since we register signal handler that calls chatkey_exit()/ if (sigfillset(&mask) < 0) return; if (sigprocmask(SIG_BLOCK, &mask, NULL) < 0) return; - /* Output new edge # and path hash. */ - fprintf(coverage_fp, "%d\n", new_edge_cnt); -#ifdef TARGET_X86_64 - fprintf(coverage_fp, "%lu\n", path_hash); - fprintf(coverage_fp, "%lu\n", edge_set_hash); -#else - fprintf(coverage_fp, "%u\n", path_hash); - fprintf(coverage_fp, "%u\n", edge_set_hash); -#endif - - fclose(coverage_fp); + if (coverage_fp) { + fprintf(coverage_fp, "%d\n%d\n", found_new_edge, found_new_path); + fclose(coverage_fp); + coverage_fp = NULL; + } - if (dbg_fp) + if (dbg_fp) { fclose(dbg_fp); + dbg_fp = NULL; + } } -static inline void chatkey_update_path_hash(register abi_ulong addr) { - register unsigned int i; - for (i = 0; i < sizeof(abi_ulong); i++) - path_hash = ((path_hash << 5) + path_hash) + ((addr >> (i<<3)) & 0xff); -} - -static inline void update_edge_set_hash(register abi_ulong edge_hash) { - edge_set_hash = edge_set_hash ^ edge_hash; -} +extern "C" void chatkey_log_bb(abi_ulong addr) { + abi_ulong edge, hash; + unsigned int byte_idx, byte_mask; + unsigned char old_byte, new_byte; -extern "C" void chatkey_log_bb(abi_ulong addr, abi_ulong callsite) { - abi_ulong edge, hash; - unsigned int byte_idx, byte_mask; - unsigned char old_byte, new_byte; + if (!coverage_fp) + return; - chatkey_update_path_hash(addr); - edge = addr; #ifdef TARGET_X86_64 - edge = (prev_node << 16) ^ addr; + edge = (prev_node << 16) ^ addr; #else - edge = (prev_node << 8) ^ addr; + edge = (prev_node << 8) ^ addr; #endif - prev_node = addr; - - if (passed_forkserver) { - // Check and update both edge_bitmap and accumulative bitmap - hash = (edge >> 4) ^ (edge << 8); - byte_idx = (hash >> 3) & 0xffff; - byte_mask = 1 << (hash & 0x7); // Use the lowest 3 bits to shift - old_byte = edge_bitmap[byte_idx]; - new_byte = old_byte | byte_mask; - if (old_byte != new_byte) { - edge_bitmap[byte_idx] = new_byte; - // If it's a new edge, update edge hash and also add to accumulative map - update_edge_set_hash(hash); - old_byte = accum_edge_bitmap[byte_idx]; - new_byte = old_byte | byte_mask; - if (old_byte != new_byte) { - new_edge_cnt++; - accum_edge_bitmap[byte_idx] = new_byte; - /* Log visited addrs if dbg_fp is not NULL */ - if (dbg_fp) { + prev_node = addr; + + // Update bitmap. + hash = (edge >> 4) ^ (edge << 8); + byte_idx = (hash >> 3) & 0xffff; + byte_mask = 1 << (hash & 0x7); // Use the lowest 3 bits to shift + old_byte = accum_edge_bitmap[byte_idx]; + new_byte = old_byte | byte_mask; + if (old_byte != new_byte) { + found_new_edge = 1; + accum_edge_bitmap[byte_idx] = new_byte; + /* Log visited addrs if dbg_fp is not NULL */ + if (dbg_fp) { #ifdef TARGET_X86_64 - fprintf(dbg_fp, "(0x%lx, 0x%lx)\n", addr, callsite); + fprintf(dbg_fp, "(0x%lx)\n", addr); #else - fprintf(dbg_fp, "(0x%x, 0x%x)\n", addr, callsite); + fprintf(dbg_fp, "(0x%x)\n", addr); #endif - } - } - } } + } } diff --git a/Instrumentor/qemu/patches-coverage/cpu-exec.diff b/Instrumentor/qemu/patches-coverage/cpu-exec.diff index 4005216..bea08c3 100644 --- a/Instrumentor/qemu/patches-coverage/cpu-exec.diff +++ b/Instrumentor/qemu/patches-coverage/cpu-exec.diff @@ -7,7 +7,7 @@ +extern abi_ulong chatkey_entry_point; /* ELF entry point (_start) */ +extern void chatkey_setup_before_forkserver(void); +extern void chatkey_setup_after_forkserver(void); -+extern void chatkey_log_bb(abi_ulong addr, abi_ulong callsite); ++extern void chatkey_log_bb(abi_ulong addr); + +#include "afl-qemu-cpu-inl.h" + @@ -37,7 +37,7 @@ + chatkey_setup_after_forkserver(); + } + -+ chatkey_log_bb(tb->pc, 0); ++ chatkey_log_bb(tb->pc); + if (qemu_loglevel_mask(CPU_LOG_EXEC)) { qemu_log("Trace %p [" TARGET_FMT_lx "] %s\n", diff --git a/Instrumentor/qemu/repatch.sh b/Instrumentor/qemu/repatch.sh index cc373b2..d58ecac 100755 --- a/Instrumentor/qemu/repatch.sh +++ b/Instrumentor/qemu/repatch.sh @@ -42,7 +42,6 @@ export_coverage_patch() { cp qemu-2.3.0-coverage/Makefile.target $TARG_DIR/Makefile.target cp qemu-2.3.0-coverage/chatkey.cc $TARG_DIR/ cp qemu-2.3.0-coverage/afl-qemu-cpu-inl.h $TARG_DIR/ - cp qemu-2.3.0-coverage/chatkey-utils.h $TARG_DIR/ } export_branch_patch() { @@ -120,7 +119,6 @@ patch -p0 ] extern void set_env (string env_variable, string env_value) -[] extern void initialize_exec (TimeoutHandling is_replay) +[] extern void initialize_exec () [] extern int init_forkserver_coverage (int argc, string[] argv, uint64 timeout) [] extern int init_forkserver_branch (int argc, string[] argv, uint64 timeout) [] extern void kill_forkserver () [] extern Signal exec (int argc, string[] argv, int stdin_size, byte[] stdin_data, uint64 timeout, bool use_pty) [] extern Signal exec_fork_coverage (uint64 timeout, int stdin_size, byte[] stdin_data) -[] extern Signal exec_fork_branch (uint64 timeout, int stdin_size, byte[] stdin_data) -[] extern int prepare_shared_mem () -[] extern int release_shared_mem () +[] extern Signal exec_fork_branch (uint64 timeout, int stdin_size, byte[] stdin_data, uint64 targ_addr, uint32 targ_index, int measure_cov) (*** Tracer and file paths ***) @@ -53,9 +41,9 @@ let selectTracer tracer arch = | BBCount, X86 -> bbCountTracerX86 | BBCount, X64 -> bbCountTracerX64 -let mutable pathHashLog = "" -let mutable branchTraceLog = "" +let mutable branchLog = "" let mutable coverageLog = "" +let mutable bitmapLog = "" let mutable dbgLog = "" let mutable forkServerEnabled = false @@ -63,24 +51,21 @@ let initialize opt = let outDir = opt.OutDir let verbosity = opt.Verbosity // Set environment variables for the instrumentor. - pathHashLog <- System.IO.Path.Combine(outDir, ".path_hash") - branchTraceLog <- System.IO.Path.Combine(outDir, ".branch_trace") + branchLog <- System.IO.Path.Combine(outDir, ".branch") coverageLog <- System.IO.Path.Combine(outDir, ".coverage") - dbgLog <- System.IO.Path.Combine(outDir, ".debug_msg") - set_env("CK_HASH_LOG", System.IO.Path.GetFullPath(pathHashLog)) - set_env("CK_FEED_LOG", System.IO.Path.GetFullPath(branchTraceLog)) + bitmapLog <- System.IO.Path.Combine(outDir, ".bitmap") + dbgLog <- System.IO.Path.Combine(outDir, ".debug") + set_env("CK_FEED_LOG", System.IO.Path.GetFullPath(branchLog)) set_env("CK_COVERAGE_LOG", System.IO.Path.GetFullPath(coverageLog)) + use bitmapFile = File.Create(bitmapLog) + bitmapFile.SetLength(0x10000L) + set_env("CK_BITMAP_LOG", System.IO.Path.GetFullPath(bitmapLog)) if verbosity >= 2 then set_env("CK_DBG_LOG", System.IO.Path.GetFullPath(dbgLog)) - // Set other environment variables in advance, to avoid affecting path hash. set_env("CK_FEED_ADDR", "0") set_env("CK_FEED_IDX", "0") - set_env("CK_CTX_SENSITIVITY", string CtxSensitivity) - initialize_exec TimeoutHandling.SendSigterm - // Initialize shared memory. - let shmID = prepare_shared_mem() - if shmID < 0 then failwith "Failed to allcate shared memory" - set_env("CK_SHM_ID", string shmID) + set_env("CK_MEASURE_COV", "0") + initialize_exec () // Initialize fork server. forkServerEnabled <- true set_env("CK_FORK_SERVER", "1") @@ -98,10 +83,9 @@ let initialize opt = let cleanup () = if forkServerEnabled then kill_forkserver () - if release_shared_mem() < 0 then log "Failed to release shared memory!" - removeFile pathHashLog - removeFile branchTraceLog + removeFile branchLog removeFile coverageLog + removeFile bitmapLog removeFile dbgLog let abandonForkServer () = @@ -121,11 +105,6 @@ let private setupFile seed = | StdInput -> () | FileInput filePath -> writeFile filePath (Seed.concretize seed) -let private clearFile seed = - match seed.Source with - | StdInput -> () - | FileInput filePath -> removeFile filePath - let private prepareStdIn seed = match seed.Source with | StdInput -> Seed.concretize seed @@ -133,39 +112,17 @@ let private prepareStdIn seed = (*** Tracer result parsing functions ***) +/// TODO. Currently we only support edge coverage gain. Will extend the system +/// to support path coverage gain if needed. let private parseCoverage filename = - let content = readAllLines filename - match content with - | [newEdgeCnt; pathHash; edgeHash] -> - int newEdgeCnt, uint64 pathHash, uint64 edgeHash - | _ -> log "[Warning] Coverage logging failed : %A" content; (0, 0UL, 0UL) - -let private parseExecHash filename = - let content = readAllLines filename - match content with - | [hash] -> uint64 hash - | _ -> log "[Warning] Coverage logging failed : %A" content; 0UL + match readAllLines filename with + | [newEdgeFlag; _] -> if int newEdgeFlag = 1 then NewEdge else NoGain + | x -> log "[Warning] Coverage logging failed : %A" x; NoGain let private is64Bit = function | X86 -> false | X64 -> true -let private readEdgeSet opt filename = - try - let f = File.Open(filename, FileMode.Open, FileAccess.Read, FileShare.Read) - use r = new BinaryReader(f) - let arch = opt.Architecture - let mutable valid = true - let edgeList = - [ while valid do - let addr = try if is64Bit arch - then r.ReadUInt64() - else uint64 (r.ReadUInt32()) - with :? EndOfStreamException -> 0UL - if addr = 0UL then valid <- false else yield addr ] - Set.ofList edgeList - with | :? FileNotFoundException -> Set.empty - let private parseBranchTraceLog opt (r:BinaryReader) tryVal = let arch = opt.Architecture let addr = @@ -206,6 +163,12 @@ let private readBranchTrace opt filename tryVal = | Some branchInfo -> yield branchInfo ] with | :? FileNotFoundException -> [] +let private tryReadBranchInfo opt filename tryVal = + match readBranchTrace opt filename tryVal with + | [] -> None + | [ branchInfo ] -> Some branchInfo + | _ -> None + (*** Tracer execute functions ***) let private runTracer tracerType opt (stdin: byte array) = @@ -218,63 +181,68 @@ let private runTracer tracerType opt (stdin: byte array) = let argc = args.Length exec(argc, args, stdin.Length, stdin, timeout, usePty) -let private runCoverageTracerForked opt (stdin: byte array) = +let private runCoverageTracerForked opt stdin = let timeout = opt.ExecTimeout - let signal = exec_fork_coverage(timeout, stdin.Length, stdin) + let stdLen = Array.length stdin + let signal = exec_fork_coverage(timeout, stdLen, stdin) if signal = Signal.ERROR then abandonForkServer () signal -let private runBranchTracerForked opt (stdin: byte array) = +let private runBranchTracerForked opt stdin addr idx measureCov = let timeout = opt.ExecTimeout - let signal = exec_fork_branch(timeout, stdin.Length, stdin) + let stdLen = Array.length stdin + let covFlag = if measureCov then 1 else 0 + let signal = exec_fork_branch(timeout, stdLen, stdin, addr, idx, covFlag) if signal = Signal.ERROR then abandonForkServer () signal +let private setEnvForBranch (addr: uint64) (idx: uint32) measureCov = + set_env("CK_FEED_ADDR", sprintf "%016x" addr) + set_env("CK_FEED_IDX", sprintf "%016x" idx) + set_env("CK_MEASURE_COV", sprintf "%d" (if measureCov then 1 else 0)) + (*** Top-level tracer executor functions ***) let getCoverage opt seed = setupFile seed let stdin = prepareStdIn seed - let exitSig = if forkServerEnabled - then runCoverageTracerForked opt stdin + let exitSig = if forkServerEnabled then runCoverageTracerForked opt stdin else runTracer Coverage opt stdin - let newEdgeCnt, pathHash, edgeHash = parseCoverage coverageLog - clearFile seed - (newEdgeCnt, pathHash, edgeHash, exitSig) + let coverageGain = parseCoverage coverageLog + (exitSig, coverageGain) let getBranchTrace opt seed tryVal = - set_env("CK_FEED_ADDR", "0") - set_env("CK_FEED_IDX", "0") setupFile seed let stdin = prepareStdIn seed - if forkServerEnabled then runBranchTracerForked opt stdin - else runTracer Branch opt stdin - |> ignore - let pathHash = parseExecHash pathHashLog - let branchTrace = readBranchTrace opt branchTraceLog tryVal - clearFile seed - removeFile pathHashLog - removeFile branchTraceLog - pathHash, branchTrace - -let getBranchInfoAt opt seed tryVal targPoint = - set_env("CK_FEED_ADDR", sprintf "%016x" targPoint.Addr) - set_env("CK_FEED_IDX", sprintf "%016x" targPoint.Idx) + let exitSig = + if forkServerEnabled then runBranchTracerForked opt stdin 0UL 0ul true + else setEnvForBranch 0UL 0ul true; runTracer Branch opt stdin + let coverageGain = parseCoverage coverageLog + let branchTrace = readBranchTrace opt branchLog tryVal + removeFile coverageLog + (exitSig, coverageGain, branchTrace) + +let getBranchInfo opt seed tryVal targPoint = setupFile seed let stdin = prepareStdIn seed - if forkServerEnabled then runBranchTracerForked opt stdin - else runTracer Branch opt stdin + let addr, idx = targPoint.Addr, uint32 targPoint.Idx + let exitSig = + if forkServerEnabled then runBranchTracerForked opt stdin addr idx true + else setEnvForBranch addr idx true; runTracer Branch opt stdin + let coverageGain = parseCoverage coverageLog + let branchInfoOpt = tryReadBranchInfo opt branchLog tryVal + removeFile coverageLog + (exitSig, coverageGain, branchInfoOpt) + +let getBranchInfoOnly opt seed tryVal targPoint = + setupFile seed + let stdin = prepareStdIn seed + let addr, idx = targPoint.Addr, uint32 targPoint.Idx + if forkServerEnabled then runBranchTracerForked opt stdin addr idx false + else setEnvForBranch addr idx false; runTracer Branch opt stdin |> ignore - let pathHash = parseExecHash pathHashLog - let branchInfoOpt = - match readBranchTrace opt branchTraceLog tryVal with - | [] -> None - | [ branchInfo ] -> Some branchInfo - | branchInfos -> None - clearFile seed - removeFile pathHashLog - removeFile branchTraceLog - (pathHash, branchInfoOpt) + let brInfoOpt = tryReadBranchInfo opt branchLog tryVal + brInfoOpt let nativeExecute opt seed = let targetProg = opt.TargetProg @@ -285,6 +253,5 @@ let nativeExecute opt seed = let cmdLine = opt.Arg.Split(WHITES, StringSplitOptions.RemoveEmptyEntries) let args = Array.append [| targetProg |] cmdLine let argc = args.Length - let signal = exec(argc, args, stdin.Length, stdin, timeout, usePty) - clearFile seed - signal + exec(argc, args, stdin.Length, stdin, timeout, usePty) + diff --git a/src/Core/Options.fs b/src/Core/Options.fs index 6642d79..a154cf2 100644 --- a/src/Core/Options.fs +++ b/src/Core/Options.fs @@ -71,7 +71,7 @@ let parseFuzzOption (args: string array) = Timelimit = r.GetResult (<@ Timelimit @>) // Options related to program execution TargetProg = System.IO.Path.GetFullPath(r.GetResult (<@ Program @>)) - ExecTimeout = r.GetResult (<@ ExecTimeout @>, defaultValue = DefaultExecTO) + ExecTimeout = r.GetResult (<@ ExecTimeout @>, defaultValue = DEF_EXEC_TO) UsePty = r.Contains (<@ UsePty @>) Architecture = r.GetResult(<@ Architecture @>, defaultValue = "X64") |> Arch.ofString diff --git a/src/Core/Seed.fs b/src/Core/Seed.fs index c9510f4..875cd65 100644 --- a/src/Core/Seed.fs +++ b/src/Core/Seed.fs @@ -95,8 +95,8 @@ module Seed = // Auxiliary function for queryUpdateBound() let private queryUpdateBoundLeft (byteVals: ByteVal []) byteCursor = let byteVals' = - if byteCursor - MaxChunkLen >= 0 - then byteVals.[byteCursor - MaxChunkLen .. byteCursor] + if byteCursor - MAX_CHUNK_LEN >= 0 + then byteVals.[byteCursor - MAX_CHUNK_LEN .. byteCursor] else byteVals.[ .. byteCursor] // We use an heuristic to bound update until the adjacent *fixed* ByteVal. match Array.tryFindIndexBack ByteVal.isFixed byteVals' with @@ -106,12 +106,12 @@ module Seed = // Auxiliary function for queryUpdateBound() let private queryUpdateBoundRight (byteVals: ByteVal []) byteCursor = let byteVals' = - if byteCursor + MaxChunkLen < byteVals.Length - then byteVals.[byteCursor .. byteCursor + MaxChunkLen] + if byteCursor + MAX_CHUNK_LEN < byteVals.Length + then byteVals.[byteCursor .. byteCursor + MAX_CHUNK_LEN] else byteVals.[byteCursor .. ] // We use an heuristic to bound update until the adjacent *fixed* ByteVal. match Array.tryFindIndex ByteVal.isFixed byteVals' with - | None -> MaxChunkLen + | None -> MAX_CHUNK_LEN | Some idx -> idx /// Find the maximum length that can be updated for grey-box concolic testing. @@ -130,10 +130,10 @@ module Seed = match direction with | Stay -> failwith "queryNeighborBytes() cannot be called with 'Stay'" | Right -> - let upperBound = min (byteVals.Length - 1) (byteCursor + MaxChunkLen) + let upperBound = min (byteVals.Length - 1) (byteCursor + MAX_CHUNK_LEN) Array.map ByteVal.getConcreteByte byteVals.[byteCursor + 1 .. upperBound] | Left -> - let lowerBound = max 0 (byteCursor - MaxChunkLen) + let lowerBound = max 0 (byteCursor - MAX_CHUNK_LEN) Array.map ByteVal.getConcreteByte byteVals.[lowerBound .. byteCursor - 1] (************************ Content update functions ************************) diff --git a/src/Core/Typedef.fs b/src/Core/Typedef.fs index 91948e0..1572485 100644 --- a/src/Core/Typedef.fs +++ b/src/Core/Typedef.fs @@ -13,6 +13,12 @@ type InputSource = /// Fuzz file input source. | FileInput of filepath: string +/// Describes the gain of coverage. +type CoverageGain = + | NoGain + | NewPath + | NewEdge + /// Priority of found seed. A seed that increased edge coverage is assigned /// 'Favored' priority, while a seed that increased path coverage is assigned /// 'Normal' priority. diff --git a/src/Core/Utils.fs b/src/Core/Utils.fs index d0566e1..4bf986f 100644 --- a/src/Core/Utils.fs +++ b/src/Core/Utils.fs @@ -11,8 +11,6 @@ let startTime = DateTime.Now let random = System.Random() -let printNewline () = Console.WriteLine "" - let printLine (str: string) = Console.WriteLine str let log fmt = @@ -20,13 +18,6 @@ let log fmt = let timeStr = "[" + elapsed.ToString("dd\:hh\:mm\:ss") + "] " Printf.kprintf (fun str -> printLine <| timeStr + str) fmt -let escapeString (str: string) = - str.Replace("\\", "\\\\").Replace("\"", "\\\"") - -let escapeWhiteSpace (str: string) = - let str = str.Replace("\n", "\\n").Replace("\r", "\\r") - str.Replace(" ", "\\s").Replace("\t", "\\t") - // Auxiliary function for splitList(). let rec private splitListAux n lst accum = match lst with @@ -105,19 +96,15 @@ let assertFileExists file = printfn "Target file ('%s') does not exist" file exit 1 -/// Remove a file, without throwing exception. -let removeFile file = - try System.IO.File.Delete(file) with _ -> () - -/// Remove a directory, without throwing exception. -let removeDir dir = - try System.IO.Directory.Delete(dir, recursive = true) with _ -> () - /// Write a file, without throwing exception. let writeFile filePath content = try System.IO.File.WriteAllBytes(filePath, content) with | _ -> log "[Warning] Failed to write file '%s'" filePath +/// Remove a file, without throwing exception. +let removeFile file = + try System.IO.File.Delete(file) with _ -> () + /// Create a directory if not exists. let createDirectoryIfNotExists dir = if not (System.IO.Directory.Exists dir) then diff --git a/src/Core/libexec.c b/src/Core/libexec.c old mode 100644 new mode 100755 index 0876791..cf8ba95 --- a/src/Core/libexec.c +++ b/src/Core/libexec.c @@ -23,30 +23,27 @@ #include #include #include -#include #include #include #include #include -#define PATH_FORKSRV_FD 198 -#define FEED_FORKSRV_FD 194 +#define COV_FORKSRV_FD 198 +#define BR_FORKSRV_FD 194 #define FORK_WAIT_MULT 10 -static int shm_id; -static pid_t path_forksrv_pid; -static int path_fsrv_ctl_fd, path_fsrv_st_fd; -static pid_t feed_forksrv_pid; -static int feed_fsrv_ctl_fd, feed_fsrv_st_fd; +static pid_t coverage_forksrv_pid; +static int coverage_fsrv_ctl_fd, coverage_fsrv_st_fd; +static pid_t branch_forksrv_pid; +static int branch_fsrv_ctl_fd, branch_fsrv_st_fd; static pid_t child_pid = 0; static int pty_fd; static int pty_flag; -static int replay_flag; static int timeout_flag; static int non_fork_stdin_fd; -static int path_stdin_fd; -static int feed_stdin_fd; +static int coverage_stdin_fd; +static int branch_stdin_fd; pid_t (*sym_forkpty)(int *, char *, const struct termios*, const struct winsize *); @@ -76,7 +73,7 @@ void open_stdin_fd(int * fd){ * descriptors up to *_FORKSRV_FD, which results in protocol error * between forkserver and its client. */ - if (*fd > PATH_FORKSRV_FD - 10) + if (*fd > COV_FORKSRV_FD - 10) error_exit("open_stdin_fd : detected a leak of file descriptor"); } @@ -97,40 +94,10 @@ static void alarm_callback(int sig) { if (child_pid) { puts("Timeout"); fflush(stdout); - if (replay_flag) { - /* If we are replaying test cases, we should use GDB to quit the - * executed program. Otherwise, gcov data will not be generated, - * and coverage cannot be obtained correctly. Codes are borrowed - * from klee_replay.c of KLEE project. - */ - int status; - char pid_buf[64]; - pid_t gdb_pid = fork(); - if (gdb_pid < 0) { - puts("[Warning] Failed to fork() for GDB execution"); - fflush(stdout); - } else if (gdb_pid == 0) { // child process - char *gdb_args[] = { - "/usr/bin/gdb", "--pid", pid_buf, "-q", "--batch", - "--eval-command=call exit(1)", NULL }; - sprintf(pid_buf, "%d", child_pid); - execvp(gdb_args[0], gdb_args); - puts("[Warning] GDB execution failed"); - fflush(stdout); - } else { // parent process, wait until GDB is executed - usleep(250 * 1000); // Give 250ms for GDB to work - /* Sometimes child process for GDB hangs, so we should send - * SIGKILL signal. */ - if ( waitpid( gdb_pid, &status, WNOHANG) < 0 ) - kill(gdb_pid, SIGKILL); - } - } else { - /* If we are in fuzzing mode, send SIGTERM (not SIGKILL) so that QEMU - * tracer can receive it and call eclipser_exit() to log feedback. - */ - kill(child_pid, SIGTERM); - } - + /* If we are in fuzzing mode, send SIGTERM (not SIGKILL) so that QEMU + * tracer can receive it and call eclipser_exit() to finish logging. + */ + kill(child_pid, SIGTERM); timeout_flag = 1; /* In some cases, the child process may not be terminated by the code * above, so examine if process is alive and send SIGKILL if so. */ @@ -140,7 +107,7 @@ static void alarm_callback(int sig) { } } -void initialize_exec (int is_replay) { +void initialize_exec(void) { struct sigaction sa; void* handle = dlopen("libutil.so.1", RTLD_LAZY); @@ -151,7 +118,6 @@ void initialize_exec (int is_replay) { sigemptyset(&sa.sa_mask); - replay_flag = is_replay; sa.sa_handler = alarm_callback; sigaction(SIGALRM, &sa, NULL); } @@ -287,8 +253,8 @@ pid_t init_forkserver(int argc, char** args, uint64_t timeout, int forksrv_fd, struct rlimit r; - if (!getrlimit(RLIMIT_NOFILE, &r) && r.rlim_cur < PATH_FORKSRV_FD + 2) { - r.rlim_cur = PATH_FORKSRV_FD + 2; + if (!getrlimit(RLIMIT_NOFILE, &r) && r.rlim_cur < COV_FORKSRV_FD + 2) { + r.rlim_cur = COV_FORKSRV_FD + 2; setrlimit(RLIMIT_NOFILE, &r); /* Ignore errors */ } @@ -364,34 +330,38 @@ pid_t init_forkserver(int argc, char** args, uint64_t timeout, int forksrv_fd, } pid_t init_forkserver_coverage(int argc, char** args, uint64_t timeout) { - path_forksrv_pid = init_forkserver(argc, args, timeout, PATH_FORKSRV_FD, - &path_stdin_fd, &path_fsrv_ctl_fd, &path_fsrv_st_fd); - return path_forksrv_pid; + coverage_forksrv_pid = init_forkserver(argc, args, timeout, COV_FORKSRV_FD, + &coverage_stdin_fd, + &coverage_fsrv_ctl_fd, + &coverage_fsrv_st_fd); + return coverage_forksrv_pid; } pid_t init_forkserver_branch(int argc, char** args, uint64_t timeout) { - feed_forksrv_pid = init_forkserver(argc, args, timeout, FEED_FORKSRV_FD, - &feed_stdin_fd, &feed_fsrv_ctl_fd, &feed_fsrv_st_fd); - return feed_forksrv_pid; + branch_forksrv_pid = init_forkserver(argc, args, timeout, BR_FORKSRV_FD, + &branch_stdin_fd, + &branch_fsrv_ctl_fd, + &branch_fsrv_st_fd); + return branch_forksrv_pid; } void kill_forkserver() { - close(path_stdin_fd); - close(feed_stdin_fd); + close(coverage_stdin_fd); + close(branch_stdin_fd); - close(path_fsrv_ctl_fd); - close(path_fsrv_st_fd); - close(feed_fsrv_ctl_fd); - close(feed_fsrv_st_fd); + close(coverage_fsrv_ctl_fd); + close(coverage_fsrv_st_fd); + close(branch_fsrv_ctl_fd); + close(branch_fsrv_st_fd); - if (path_forksrv_pid) { - kill(path_forksrv_pid, SIGKILL); - path_forksrv_pid = 0; + if (coverage_forksrv_pid) { + kill(coverage_forksrv_pid, SIGKILL); + coverage_forksrv_pid = 0; } - if (feed_forksrv_pid) { - kill(feed_forksrv_pid, SIGKILL); - feed_forksrv_pid = 0; + if (branch_forksrv_pid) { + kill(branch_forksrv_pid, SIGKILL); + branch_forksrv_pid = 0; } } @@ -401,16 +371,16 @@ int exec_fork_coverage(uint64_t timeout, int stdin_size, char *stdin_data) { static unsigned char tmp[4]; /* TODO : what if we want to use pseudo-terminal? */ - write_stdin(path_stdin_fd, stdin_size, stdin_data); + write_stdin(coverage_stdin_fd, stdin_size, stdin_data); - if ((res = write(path_fsrv_ctl_fd, tmp, 4)) != 4) { + if ((res = write(coverage_fsrv_ctl_fd, tmp, 4)) != 4) { perror("exec_fork_coverage: Cannot request new process to fork server"); printf("write() call ret = %d\n", res); return -1; } - if ((res = read(path_fsrv_st_fd, &child_pid, 4)) != 4) { - perror("exec_fork_coverage: Failed to receive child pid from fork server"); + if ((res = read(coverage_fsrv_st_fd, &child_pid, 4)) != 4) { + perror("exec_fork_coverage: Cannot receive child pid from fork server"); printf("read() call ret = %d, child_pid = %d\n", res, child_pid); return -1; } @@ -424,7 +394,7 @@ int exec_fork_coverage(uint64_t timeout, int stdin_size, char *stdin_data) { it.it_value.tv_usec = (timeout % 1000) * 1000; setitimer(ITIMER_REAL, &it, NULL); - if ((res = read(path_fsrv_st_fd, &childstatus, 4)) != 4) { + if ((res = read(coverage_fsrv_st_fd, &childstatus, 4)) != 4) { perror("exec_fork_coverage: Unable to communicate with fork server"); printf("read() call ret = %d, childstatus = %d\n", res, childstatus); return -1; @@ -450,30 +420,34 @@ int exec_fork_coverage(uint64_t timeout, int stdin_size, char *stdin_data) { } } -int exec_fork_branch(uint64_t timeout, int stdin_size, char *stdin_data) { - uint64_t targ_addr, targ_index; +int exec_fork_branch(uint64_t timeout, int stdin_size, char *stdin_data, + uint64_t targ_addr, uint32_t targ_index, int measure_cov) { int res, childstatus; static struct itimerval it; /* TODO : what if we want to use pseudo-terminal? */ - write_stdin(feed_stdin_fd, stdin_size, stdin_data); + write_stdin(branch_stdin_fd, stdin_size, stdin_data); - targ_addr = strtol(getenv("CK_FEED_ADDR"), NULL, 16); - targ_index = strtol(getenv("CK_FEED_IDX"), NULL, 16); - if ((res = write(feed_fsrv_ctl_fd, &targ_addr, 8)) != 8) { - perror("exec_fork_branch: Cannot request new process to fork server (1)"); + if ((res = write(branch_fsrv_ctl_fd, &targ_addr, 8)) != 8) { + perror("exec_fork_branch: Cannot send targ_addr to fork server"); printf("write() call ret = %d\n", res); return -1; } - if ((res = write(feed_fsrv_ctl_fd, &targ_index, 8)) != 8) { - perror("exec_fork_branch: Cannot request new process to fork server (2)"); + if ((res = write(branch_fsrv_ctl_fd, &targ_index, 4)) != 4) { + perror("exec_fork_branch: Cannot send targ_index to fork server"); printf("write() call ret = %d\n", res); return -1; } - if ((res = read(feed_fsrv_st_fd, &child_pid, 4)) != 4) { - perror("exec_fork_branch: Failed to receive child pid from fork server"); + if ((res = write(branch_fsrv_ctl_fd, &measure_cov, 4)) != 4) { + perror("exec_fork_branch: Cannot send measure_cov to fork server"); + printf("write() call ret = %d\n", res); + return -1; + } + + if ((res = read(branch_fsrv_st_fd, &child_pid, 4)) != 4) { + perror("exec_fork_branch: Cannot receive child pid from fork server"); printf("read() call ret = %d, child_pid = %d\n", res, child_pid); return -1; } @@ -487,7 +461,7 @@ int exec_fork_branch(uint64_t timeout, int stdin_size, char *stdin_data) { it.it_value.tv_usec = (timeout % 1000) * 1000; setitimer(ITIMER_REAL, &it, NULL); - if ((res = read(feed_fsrv_st_fd, &childstatus, 4)) != 4) { + if ((res = read(branch_fsrv_st_fd, &childstatus, 4)) != 4) { perror("exec_fork_branch: Unable to communicate with fork server"); printf("read() call ret = %d, childstatus = %d\n", res, childstatus); return -1; @@ -512,12 +486,3 @@ int exec_fork_branch(uint64_t timeout, int stdin_size, char *stdin_data) { return 0; } } - -int prepare_shared_mem(void) { - shm_id = shmget(IPC_PRIVATE, 0x10000, IPC_CREAT | IPC_EXCL | 0600); - return shm_id; -} - -int release_shared_mem(void) { - return shmctl(shm_id, IPC_RMID, 0); -} diff --git a/src/Eclipser.fsproj b/src/Eclipser.fsproj index cd13486..14c9c0d 100644 --- a/src/Eclipser.fsproj +++ b/src/Eclipser.fsproj @@ -14,7 +14,6 @@ - @@ -24,6 +23,7 @@ + diff --git a/src/Fuzz/Fuzz.fs b/src/Fuzz/Fuzz.fs index 4f24ff4..9df0221 100644 --- a/src/Fuzz/Fuzz.fs +++ b/src/Fuzz/Fuzz.fs @@ -4,22 +4,22 @@ open System.Threading open Utils open Options -let private printFoundSeed verbosity seed newEdgeN = - let edgeStr = if newEdgeN > 0 then sprintf "(%d new edges) " newEdgeN else "" +let private printFoundSeed verbosity seed = if verbosity >= 1 then - log "[*] Found by grey-box concolic %s: %s" edgeStr (Seed.toString seed) + log "[*] Found by grey-box concolic: %s" (Seed.toString seed) elif verbosity >= 0 then - log "[*] Found by grey-box concolic %s" edgeStr + log "[*] Found by grey-box concolic %s" (Seed.toString seed) -let private evalSeed opt seed = - let newEdgeN, pathHash, edgeHash, exitSig = Executor.getCoverage opt seed - let isNewPath = Manager.save opt seed newEdgeN pathHash edgeHash exitSig false - if newEdgeN > 0 then printFoundSeed opt.Verbosity seed newEdgeN +let private coverageToPriority = function + | NoGain -> None + | NewPath -> Some Normal + | NewEdge -> Some Favored + +let private evalSeed opt seed exitSig covGain = + Manager.save opt seed exitSig covGain + if covGain = NewEdge then printFoundSeed opt.Verbosity seed let isAbnormal = Signal.isTimeout exitSig || Signal.isCrash exitSig - if isNewPath && not isAbnormal then - let priority = if newEdgeN > 0 then Favored else Normal - Some priority - else None + if isAbnormal then None else coverageToPriority covGain let private initializeSeeds opt = if opt.InputDir = "" then [Seed.make opt.FuzzSource] @@ -29,11 +29,9 @@ let private initializeSeeds opt = |> List.map (Seed.makeWith opt.FuzzSource) // Create seed with content let private preprocessAux opt seed = - let newEdgeN, pathHash, edgeHash, exitSig = Executor.getCoverage opt seed - let isNewPath = Manager.save opt seed newEdgeN pathHash edgeHash exitSig true - if newEdgeN > 0 then Some (Favored, seed) - elif isNewPath then Some (Normal, seed) - else None + let exitSig, covGain = Executor.getCoverage opt seed + Manager.save opt seed exitSig covGain + Option.map (fun pr -> (pr, seed)) (coverageToPriority covGain) let private preprocess opt seeds = log "[*] Total %d initial seeds" (List.length seeds) @@ -44,9 +42,9 @@ let private preprocess opt seeds = log "[*] %d initial items with low priority" normalCount items -let private makeNewItems opt seeds = - let collector seed = - match evalSeed opt seed with +let private relocateItems opt seeds = + let collector (seed, exitSig, covGain) = + match evalSeed opt seed exitSig covGain with | None -> [] | Some pr -> List.map (fun s -> (pr, s)) (Seed.relocateCursor seed) List.collect collector seeds @@ -64,12 +62,12 @@ let rec private fuzzLoop opt seedQueue = let priority, seed, seedQueue = SeedQueue.dequeue seedQueue if opt.Verbosity >= 1 then log "Grey-box concolic on %A seed : %s" priority (Seed.toString seed) - let newSeeds = GreyConcolic.run seed opt + let newItems = GreyConcolic.run seed opt // Move cursors of newly generated seeds. - let newItems = makeNewItems opt newSeeds + let relocatedItems = relocateItems opt newItems // Also generate seeds by just stepping the cursor of original seed. let steppedItems = makeSteppedItems priority seed - let seedQueue = List.fold SeedQueue.enqueue seedQueue newItems + let seedQueue = List.fold SeedQueue.enqueue seedQueue relocatedItems let seedQueue = List.fold SeedQueue.enqueue seedQueue steppedItems // Perform random fuzzing fuzzLoop opt seedQueue diff --git a/src/Fuzz/Manager.fs b/src/Fuzz/Manager.fs index 40161cc..63f4673 100644 --- a/src/Fuzz/Manager.fs +++ b/src/Fuzz/Manager.fs @@ -26,25 +26,8 @@ let mutable private argTestCaseCount = 0 let mutable private stdinTestCaseCount = 0 let mutable private fileTestCaseCount = 0 let mutable private testCaseCount = 0 -let private pathHashes = new HashSet() -let private crashEdgeHashes = new HashSet() - -let isNewPath pathHash = - // The case of pathHash = 0UL means abortion due to threshold or other errors. - pathHash <> 0UL && not (pathHashes.Contains pathHash) - -let private addPathHash pathHash = - // The case of pathHash = 0UL means abortion due to threshold or other errors. - pathHash <> 0UL && pathHashes.Add pathHash - -let private addCrashHash crashEdgeHash = - crashEdgeHash <> 0UL && crashEdgeHashes.Add crashEdgeHash - -let getPathCount () = - pathHashes.Count let printStatistics () = - log "Paths : %d" pathHashes.Count log "Testcases : %d" testCaseCount log "Input vector of test cases" log " Argument : %d" argTestCaseCount @@ -83,19 +66,15 @@ let private dumpTestCase seed = System.IO.File.WriteAllBytes(tcPath, Seed.concretize seed) updateTestcaseCount () -let private checkCrash opt exitSig seed edgeHash = - if Signal.isCrash exitSig && addCrashHash edgeHash - then (true, exitSig) +let private checkCrash opt seed exitSig covGain = + if Signal.isCrash exitSig && covGain = NewEdge then (true, exitSig) elif Signal.isTimeout exitSig then // Check again with native execution let exitSig' = Executor.nativeExecute opt seed - if Signal.isCrash exitSig' && addCrashHash edgeHash - then (true, exitSig') + if Signal.isCrash exitSig' && covGain = NewEdge then (true, exitSig') else (false, exitSig') else (false, exitSig) -let save opt seed newN pathHash edgeHash exitSig isInitSeed = - let isNewPath = addPathHash pathHash - let isNewCrash, exitSig' = checkCrash opt exitSig seed edgeHash - if newN > 0 || isInitSeed then dumpTestCase seed +let save opt seed exitSig covGain = + let isNewCrash, exitSig' = checkCrash opt seed exitSig covGain if isNewCrash then dumpCrash opt seed exitSig' - isNewPath + if covGain = NewEdge then dumpTestCase seed diff --git a/src/GreyConcolic/BranchTrace.fs b/src/GreyConcolic/BranchTrace.fs index b6f90d5..b7feffa 100644 --- a/src/GreyConcolic/BranchTrace.fs +++ b/src/GreyConcolic/BranchTrace.fs @@ -7,24 +7,16 @@ type BranchTrace = BranchInfo list module BranchTrace = - let collectAux opt (accTraces, accNewPathSeeds, accPaths) seed v = - let pathHash, branchTrace = Executor.getBranchTrace opt seed v - let accTraces = branchTrace :: accTraces - let accNewPathSeeds = - if Manager.isNewPath pathHash && not (Set.contains pathHash accPaths) - then seed :: accNewPathSeeds - else accNewPathSeeds - let accPaths = Set.add pathHash accPaths - (accTraces, accNewPathSeeds, accPaths) + let collectAux opt seed tryVal = + let tryByteVal = Sampled (byte tryVal) + let trySeed = Seed.updateCurByte seed tryByteVal + let exitSig, covGain, trace = Executor.getBranchTrace opt trySeed tryVal + (trace, (trySeed, exitSig, covGain)) let collect seed opt minVal maxVal = let nSpawn = opt.NSpawn let tryVals = sampleInt minVal maxVal nSpawn - let tryBytes = List.map (fun v -> Sampled (byte v)) tryVals - let trySeeds = List.map (Seed.updateCurByte seed) tryBytes - let traces, newPathSeeds, _ = - List.fold2 (collectAux opt) ([], [], Set.empty) trySeeds tryVals - (List.rev traces, newPathSeeds) // List.rev to maintain order + List.map (collectAux opt seed) tryVals |> List.unzip let getHeadAddr (brTrace: BranchTrace) = match brTrace with diff --git a/src/GreyConcolic/BranchTree.fs b/src/GreyConcolic/BranchTree.fs index 36a08d9..0e9b3fd 100644 --- a/src/GreyConcolic/BranchTree.fs +++ b/src/GreyConcolic/BranchTree.fs @@ -77,7 +77,7 @@ module BranchTree = let inferLinEq ctx brInfos = if checkValidTarget brInfos then - genComb brInfos BranchCombinationWindow 3 // XXX + genComb brInfos BRANCH_COMB_WINDOW 3 // XXX (* Now convert each [a,b,c] into (a,b,c) *) |> List.map (function [ a; b; c ] -> (a,b,c) | _ -> failwith "invalid") |> inferLinEqAux ctx @@ -92,7 +92,7 @@ module BranchTree = let inferLinIneq ctx brInfos = if checkValidTarget brInfos then - genComb brInfos BranchCombinationWindow 3 // XXX + genComb brInfos BRANCH_COMB_WINDOW 3 // XXX (* Now convert each [a,b,c] into (a,b,c) *) |> List.map (function a::b::c::[] -> (a,b,c) | _ -> failwith "invalid") |> inferLinIneqAux ctx diff --git a/src/GreyConcolic/GreyConcolic.fs b/src/GreyConcolic/GreyConcolic.fs index c1465de..4974964 100644 --- a/src/GreyConcolic/GreyConcolic.fs +++ b/src/GreyConcolic/GreyConcolic.fs @@ -7,7 +7,7 @@ let run seed opt = let seedStr = Seed.toString seed failwithf "Cursor pointing to Fixed ByteVal %s" seedStr let minVal, maxVal = bigint (int minByte), bigint (int maxByte) - let branchTraces, spawnSeeds = BranchTrace.collect seed opt minVal maxVal + let branchTraces, byProducts = BranchTrace.collect seed opt minVal maxVal let byteDir = Seed.getByteCursorDir seed let bytes = Seed.queryNeighborBytes seed byteDir let ctx = { Bytes = bytes; ByteDir = byteDir } @@ -15,4 +15,4 @@ let run seed opt = let branchTree = BranchTree.selectAndRepair opt branchTree GreySolver.clearSolutionCache () let solutions = GreySolver.solve seed opt byteDir branchTree - solutions @ spawnSeeds + solutions @ byProducts diff --git a/src/GreyConcolic/Solve.fs b/src/GreyConcolic/Solve.fs index 0a28b18..0d9bd44 100644 --- a/src/GreyConcolic/Solve.fs +++ b/src/GreyConcolic/Solve.fs @@ -15,42 +15,40 @@ module GreySolver = | tryVal :: tailVals -> let tryStr = Array.append accStr [| byte tryVal |] let trySeed = Seed.fixCurBytes seed Right tryStr - let _, brInfoOpt = Executor.getBranchInfoAt opt trySeed tryVal targPt - match brInfoOpt with - | None -> // Failed to observe target point, proceed with the next tryVal + match Executor.getBranchInfoOnly opt trySeed tryVal targPt with + | None -> // Failed to observe target point, proceed with the next tryVal. findNextCharAux seed opt targPt accStr accBrInfos tailVals | Some brInfo -> let accBrInfos = accBrInfos @ [brInfo] let ctx = { Bytes = [| |]; ByteDir = Right } match BranchTree.inferLinEq ctx accBrInfos with - | None -> // No linear equation found yet, proceed with more brInfo + | None -> // No linear equation found yet, proceed with more brInfo. findNextCharAux seed opt targPt accStr accBrInfos tailVals - | Some linEq -> (* The solution of this equation is next character *) + | Some linEq -> // The solution of this equation is next character. match linEq.Solutions with | [] -> failwith "Linear equation w/ empty solution" | sol :: _ -> Some sol - let findNextChar seed opt targPt accStr = + let private findNextChar seed opt targPt accStr = let sampleVals = sampleInt 0I 255I opt.NSpawn findNextCharAux seed opt targPt accStr [] sampleVals let rec tryStrSol seed opt maxLen targPt accRes tryStr = let trySeed = Seed.fixCurBytes seed Right tryStr // Use dummy value as 'tryVal', since our interest is in branch distance. - match Executor.getBranchInfoAt opt trySeed 0I targPt with - | pathHash, Some brInfo when brInfo.Distance = 0I -> - if Manager.isNewPath pathHash - then (trySeed :: accRes) - else accRes - | _, Some _ -> // Non-zero branch distance, try next character. - if Array.length tryStr >= maxLen then accRes else + match Executor.getBranchInfo opt trySeed 0I targPt with + | exitSig, covGain, Some brInfo when brInfo.Distance = 0I -> + ((trySeed, exitSig, covGain) :: accRes) + | _, _, Some _ -> // Non-zero branch distance, try next character. + if Array.length tryStr >= maxLen then accRes + else let nextCharOpt = findNextChar seed opt targPt tryStr match nextCharOpt with | None -> accRes | Some nextChar -> let tryStr = Array.append tryStr [| byte nextChar |] tryStrSol seed opt maxLen targPt accRes tryStr - | _, None -> accRes // Target point disappeared, halt. + | _, _, None -> accRes // Target point disappeared, halt. let solveAsString seed opt targPt linEq accRes = let initStrs = List.map (bigIntToBytes BE 1) linEq.Solutions @@ -68,14 +66,12 @@ module GreySolver = let tryBytes = bigIntToBytes endian size sol let trySeed = Seed.fixCurBytes seed dir tryBytes // Use dummy value as 'tryVal', since our interest is branch distance. - match Executor.getBranchInfoAt opt trySeed 0I targPt with - | pathHash, Some brInfo when brInfo.Distance = 0I -> + match Executor.getBranchInfo opt trySeed 0I targPt with + | exitSig, covGain, Some brInfo when brInfo.Distance = 0I -> ignore (solutionCache.Add(sol)) - if Manager.isNewPath pathHash - then trySeed :: accRes - else accRes - | _, Some _ -> accRes // Non-zero branch distance, failed. - | _, None -> accRes // Target point disappeared, halt. + (trySeed, exitSig, covGain) :: accRes + | _, _, Some _ -> accRes // Non-zero branch distance, failed. + | _, _, None -> accRes // Target point disappeared, failed. let solveAsChunk seed opt dir targPt linEq accRes = let sols = linEq.Solutions @@ -88,12 +84,6 @@ module GreySolver = then solveAsString seed opt targPt linEq accRes else solveAsChunk seed opt dir targPt linEq accRes - let solveEquations seed opt dir linEqs = - let solveN = opt.NSolve / 3 - let linEqsChosen = randomSelect linEqs solveN - List.fold (solveEquation seed opt dir) [] linEqsChosen - |> List.rev // To preserve original order - (* Functions related to binary search on monotonic function *) let getFunctionValue monotonic brInfo = @@ -112,19 +102,17 @@ module GreySolver = let endian = if dir = Left then LE else BE let tryBytes = bigIntToBytes endian mono.ByteLen tryVal let trySeed = Seed.fixCurBytes seed dir tryBytes - match Executor.getBranchInfoAt opt trySeed tryVal targPt with - | _, None -> accRes // Target point disappeared, halt. - | pathHash, Some brInfo when brInfo.Distance = 0I -> - if Manager.isNewPath pathHash - then trySeed :: accRes - else accRes - | _, Some brInfo -> // Caution : In this case, pathHash is incorrect + match Executor.getBranchInfo opt trySeed tryVal targPt with + | exitSig, covGain, Some brInfo when brInfo.Distance = 0I -> + (trySeed, exitSig, covGain) :: accRes + | _, _, Some brInfo -> let newY = getFunctionValue mono brInfo (* TODO : check monotonicity violation, too. *) let newMono = Monotonicity.update mono tryVal newY if newMono.ByteLen <= maxLen then binarySearch seed opt dir maxLen targPt accRes newMono else accRes + | _, _, None -> accRes // Target point disappeared, halt. let solveMonotonic seed opt accRes (targPt, mono) = let maxLenR = Seed.queryUpdateBound seed Right @@ -134,12 +122,6 @@ module GreySolver = if not (List.isEmpty res) then res @ accRes else binarySearch seed opt Left maxLenL targPt accRes mono - let solveMonotonics seed opt monotonics = - let solveN = opt.NSolve / 3 - let monosChosen = randomSelect monotonics solveN - List.fold (solveMonotonic seed opt) [] monosChosen - |> List.rev // To preserve original order - (* Functions related to solving linear inequalities *) let differentSign i1 i2 = i1 < 0I && i2 > 0I || i1 > 0I && i2 < 0I @@ -187,16 +169,16 @@ module GreySolver = let tryBytes = bigIntToBytes endian size sol let trySeed = Seed.fixCurBytes seed dir tryBytes // Use dummy value as 'tryVal', since our interest is in branch distance. - match Executor.getBranchInfoAt opt trySeed 0I targPt with - | _, Some brInfo when brInfo.Distance = 0I -> + match Executor.getBranchInfoOnly opt trySeed 0I targPt with + | Some brInfo when brInfo.Distance = 0I -> let tryBytes' = bigIntToBytes endian size (sol - 1I) let trySeed' = Seed.fixCurBytes seed dir tryBytes' - match Executor.getBranchInfoAt opt trySeed' 0I targPt with - | _, Some brInfo' -> + match Executor.getBranchInfoOnly opt trySeed' 0I targPt with + | Some brInfo' -> let sign = if brInfo'.Distance > 0I then Positive else Negative (sol, sign) :: accRes - | _ -> accRes - | _, _ -> accRes + | None -> accRes + | _ -> accRes let checkSolution seed opt dir equation targPt = let solutions = equation.Solutions @@ -211,8 +193,8 @@ module GreySolver = let tryBytes2 = bigIntToBytes endian size sol2 let trySeed2 = Seed.fixCurBytes seed dir tryBytes2 // Use dummy value as 'tryVal', since our interest is in branch distance. - let _, brInfoOpt1 = Executor.getBranchInfoAt opt trySeed1 0I targPt - let _, brInfoOpt2 = Executor.getBranchInfoAt opt trySeed2 0I targPt + let brInfoOpt1 = Executor.getBranchInfoOnly opt trySeed1 0I targPt + let brInfoOpt2 = Executor.getBranchInfoOnly opt trySeed2 0I targPt match brInfoOpt1, brInfoOpt2 with | Some brInfo1, Some brInfo2 -> if sameSign brInfo1.Distance brInfo2.Distance @@ -299,14 +281,16 @@ module GreySolver = let cond, branchPoint = branchCond match cond with | LinEq linEq -> - let seeds = solveEquation seed opt dir [] (branchPoint, linEq) - (pc, seeds) + let items = solveEquation seed opt dir [] (branchPoint, linEq) + (pc, items) | Mono mono -> - let seeds = solveMonotonic seed opt [] (branchPoint, mono) - (pc, seeds) + let items = solveMonotonic seed opt [] (branchPoint, mono) + (pc, items) | LinIneq ineq -> let pc, seeds = solveInequality seed opt dir pc distSign branchPoint ineq - (pc, seeds) + let sigs, covs = List.map (Executor.getCoverage opt) seeds |> List.unzip + let items = List.zip3 seeds sigs covs + (pc, items) let solveBranchSeq seed opt dir pc branchSeq = List.fold (fun (accPc, accSeeds) branch -> @@ -319,7 +303,10 @@ module GreySolver = | Straight branchSeq -> let pc, newSeeds = solveBranchSeq seed opt dir pc branchSeq let terminalSeeds = encodeCondition seed opt dir pc - newSeeds @ terminalSeeds + let results = List.map (Executor.getCoverage opt) terminalSeeds + let sigs, covs = List.unzip results + let terminalItems = List.zip3 terminalSeeds sigs covs + newSeeds @ terminalItems | ForkedTree (branchSeq, (LinIneq ineq, branchPt), childs) -> let pc, newSeeds = solveBranchSeq seed opt dir pc branchSeq let condP, condN = extractCond seed opt dir ineq branchPt From c77c759fb4697facde62f1857db7ec01e0a0f668 Mon Sep 17 00:00:00 2001 From: Jaeseung Choi Date: Thu, 24 Sep 2020 19:15:02 +0900 Subject: [PATCH 08/29] Cleanup and refactor code --- src/Eclipser.fsproj | 2 +- src/Fuzz/Fuzz.fs | 25 ++++++++++--------------- src/Fuzz/{Manager.fs => TestCase.fs} | 10 +--------- src/GreyConcolic/Solve.fs | 2 +- 4 files changed, 13 insertions(+), 26 deletions(-) rename src/Fuzz/{Manager.fs => TestCase.fs} (88%) diff --git a/src/Eclipser.fsproj b/src/Eclipser.fsproj index 14c9c0d..e3a84fa 100644 --- a/src/Eclipser.fsproj +++ b/src/Eclipser.fsproj @@ -23,7 +23,7 @@ - + diff --git a/src/Fuzz/Fuzz.fs b/src/Fuzz/Fuzz.fs index 9df0221..a419691 100644 --- a/src/Fuzz/Fuzz.fs +++ b/src/Fuzz/Fuzz.fs @@ -16,7 +16,7 @@ let private coverageToPriority = function | NewEdge -> Some Favored let private evalSeed opt seed exitSig covGain = - Manager.save opt seed exitSig covGain + TestCase.save opt seed exitSig covGain if covGain = NewEdge then printFoundSeed opt.Verbosity seed let isAbnormal = Signal.isTimeout exitSig || Signal.isCrash exitSig if isAbnormal then None else coverageToPriority covGain @@ -30,19 +30,14 @@ let private initializeSeeds opt = let private preprocessAux opt seed = let exitSig, covGain = Executor.getCoverage opt seed - Manager.save opt seed exitSig covGain + TestCase.save opt seed exitSig covGain Option.map (fun pr -> (pr, seed)) (coverageToPriority covGain) let private preprocess opt seeds = log "[*] Total %d initial seeds" (List.length seeds) - let items = List.choose (preprocessAux opt) seeds - let favoredCount = List.filter (fst >> (=) Favored) items |> List.length - let normalCount = List.filter (fst >> (=) Normal) items |> List.length - log "[*] %d initial items with high priority" favoredCount - log "[*] %d initial items with low priority" normalCount - items + List.choose (preprocessAux opt) seeds -let private relocateItems opt seeds = +let private makeRelocatedItems opt seeds = let collector (seed, exitSig, covGain) = match evalSeed opt seed exitSig covGain with | None -> [] @@ -63,13 +58,13 @@ let rec private fuzzLoop opt seedQueue = if opt.Verbosity >= 1 then log "Grey-box concolic on %A seed : %s" priority (Seed.toString seed) let newItems = GreyConcolic.run seed opt - // Move cursors of newly generated seeds. - let relocatedItems = relocateItems opt newItems - // Also generate seeds by just stepping the cursor of original seed. + // Relocate the cursors of newly generated seeds. + let relocatedItems = makeRelocatedItems opt newItems + // Also generate seeds by just stepping the cursor of the original seed. let steppedItems = makeSteppedItems priority seed + // Add the new items to the seed queue. let seedQueue = List.fold SeedQueue.enqueue seedQueue relocatedItems let seedQueue = List.fold SeedQueue.enqueue seedQueue steppedItems - // Perform random fuzzing fuzzLoop opt seedQueue let private fuzzingTimer timeoutSec = async { @@ -77,7 +72,7 @@ let private fuzzingTimer timeoutSec = async { System.Threading.Thread.Sleep(timespan ) printLine "Fuzzing timeout expired." log "===== Statistics =====" - Manager.printStatistics () + TestCase.printStatistics () log "Done, clean up and exit..." Executor.cleanup () exit (0) @@ -91,7 +86,7 @@ let main args = log "[*] Fuzz target : %s" opt.TargetProg log "[*] Time limit : %d sec" opt.Timelimit createDirectoryIfNotExists opt.OutDir - Manager.initialize opt.OutDir + TestCase.initialize opt.OutDir Executor.initialize opt let emptyQueue = SeedQueue.initialize () let initialSeeds = initializeSeeds opt diff --git a/src/Fuzz/Manager.fs b/src/Fuzz/TestCase.fs similarity index 88% rename from src/Fuzz/Manager.fs rename to src/Fuzz/TestCase.fs index 63f4673..a9a439d 100644 --- a/src/Fuzz/Manager.fs +++ b/src/Fuzz/TestCase.fs @@ -1,6 +1,5 @@ -module Eclipser.Manager +module Eclipser.TestCase -open System.Collections.Generic open Utils open Options @@ -22,17 +21,10 @@ let mutable private illegalInstrCount = 0 let mutable private fpErrorCount = 0 let mutable private abortCount = 0 let mutable private crashCount = 0 -let mutable private argTestCaseCount = 0 -let mutable private stdinTestCaseCount = 0 -let mutable private fileTestCaseCount = 0 let mutable private testCaseCount = 0 let printStatistics () = log "Testcases : %d" testCaseCount - log "Input vector of test cases" - log " Argument : %d" argTestCaseCount - log " Stdin : %d" stdinTestCaseCount - log " File : %d" fileTestCaseCount log "Crashes : %d" crashCount log " Segfault : %d" segfaultCount log " Illegal instruction : %d" illegalInstrCount diff --git a/src/GreyConcolic/Solve.fs b/src/GreyConcolic/Solve.fs index 0d9bd44..e4bba2c 100644 --- a/src/GreyConcolic/Solve.fs +++ b/src/GreyConcolic/Solve.fs @@ -107,7 +107,7 @@ module GreySolver = (trySeed, exitSig, covGain) :: accRes | _, _, Some brInfo -> let newY = getFunctionValue mono brInfo - (* TODO : check monotonicity violation, too. *) + // TODO : check monotonicity violation, too. let newMono = Monotonicity.update mono tryVal newY if newMono.ByteLen <= maxLen then binarySearch seed opt dir maxLen targPt accRes newMono From 431c606db0ee6be099aaee139a710f97a81551d6 Mon Sep 17 00:00:00 2001 From: Jaeseung Choi Date: Fri, 25 Sep 2020 13:40:26 +0900 Subject: [PATCH 09/29] Support synchronization with AFL Now rely on AFL for random mutation. --- src/Core/Options.fs | 54 ++++++++++++++++++++++++-------------------- src/Core/Typedef.fs | 8 +++++++ src/Eclipser.fsproj | 1 + src/Fuzz/Fuzz.fs | 41 ++++++++++++++++----------------- src/Fuzz/Sync.fs | 49 ++++++++++++++++++++++++++++++++++++++++ src/Fuzz/TestCase.fs | 4 ++-- 6 files changed, 109 insertions(+), 48 deletions(-) create mode 100644 src/Fuzz/Sync.fs diff --git a/src/Core/Options.fs b/src/Core/Options.fs index a154cf2..325d828 100644 --- a/src/Core/Options.fs +++ b/src/Core/Options.fs @@ -5,38 +5,40 @@ open Utils open Config type FuzzerCLI = - | [] [] [] Program of path: string | [] [] Verbose of int | [] [] [] Timelimit of sec: int | [] [] [] OutputDir of path: string - // Options related to seed initialization - | [] [] InputDir of path: string - | [] Arg of string - | [] [] Filepath of string - // Options related to execution of program + | [] [] SyncDir of path: string + // Options related to program execution. + | [] [] [] Program of path: string | [] ExecTimeout of millisec:uint64 | [] UsePty | [] Architecture of string - // Options related to fuzzing process + // Options related to seed. + | [] [] InputDir of path: string + | [] Arg of string + | [] [] Filepath of string + // Options related to grey-box concolic testing technique. | [] NSolve of int | [] NSpawn of int with interface IArgParserTemplate with member s.Usage = match s with - | Program _ -> "Target program for test case generation with fuzzing." | Verbose _ -> "Verbosity level to control debug messages (default:0)." | Timelimit _ -> "Timeout for fuzz testing (in seconds)." | OutputDir _ -> "Directory to store testcase outputs." - | InputDir _ -> "Directory containing initial seeds." - // Options related to seed initialization - | Arg _ -> "Command-line argument of program under test." - | Filepath _ -> "File input's (fixed) path" - // Options related to execution of program + | SyncDir _ -> "Directory shared with AFL instances" + // Options related to program execution. + | Program _ -> "Target program for test case generation with fuzzing." | ExecTimeout _ -> "Execution timeout (ms) for a fuzz run (default:500)" | UsePty _ -> "Use pseudo tty for standard input" | Architecture _ -> "Target program architecture (X86|X64) (default:X64)" - // Options related to test case generation + // Options related to seed. + | InputDir _ -> "Directory containing initial seeds." + | Arg _ -> "Command-line argument of program under test." + | Filepath _ -> "File input's (fixed) path" + // Options related to grey-box concolic testing technique. | NSolve _ -> "Number of branches to flip in grey-box concolic testing " + "when an execution path is given. 'N_solve' parameter in " + "the paper." @@ -45,18 +47,19 @@ with type FuzzOption = { Verbosity : int - OutDir : string Timelimit : int - // Options related to program execution + OutDir : string + SyncDir : string + // Options related to program execution. TargetProg : string ExecTimeout : uint64 UsePty : bool Architecture : Arch - // Options related to seed - FuzzSource : InputSource + // Options related to seed. InputDir : string Arg : string - // Options related to test case generation + FuzzSource : InputSource + // Options related to grey-box concolic testing technique. NSolve : int NSpawn : int } @@ -67,20 +70,21 @@ let parseFuzzOption (args: string array) = let r = try parser.Parse(args) with :? Argu.ArguParseException -> printLine (parser.PrintUsage()); exit 1 { Verbosity = r.GetResult (<@ Verbose @>, defaultValue = 0) - OutDir = r.GetResult (<@ OutputDir @>) Timelimit = r.GetResult (<@ Timelimit @>) - // Options related to program execution + OutDir = r.GetResult (<@ OutputDir @>) + SyncDir = r.GetResult (<@ SyncDir @>, defaultValue = "") + // Options related to program execution. TargetProg = System.IO.Path.GetFullPath(r.GetResult (<@ Program @>)) ExecTimeout = r.GetResult (<@ ExecTimeout @>, defaultValue = DEF_EXEC_TO) UsePty = r.Contains (<@ UsePty @>) Architecture = r.GetResult(<@ Architecture @>, defaultValue = "X64") |> Arch.ofString - // Options related to seed - FuzzSource = if not (r.Contains(<@ Filepath @>)) then StdInput - else FileInput (r.GetResult (<@ Filepath @>)) + // Options related to seed. InputDir = r.GetResult(<@ InputDir @>, defaultValue = "") Arg = r.GetResult (<@ Arg @>, defaultValue = "") - // Options related to test case generation + FuzzSource = if not (r.Contains(<@ Filepath @>)) then StdInput + else FileInput (r.GetResult (<@ Filepath @>)) + // Options related to grey-box concolic testing technique. NSolve = r.GetResult(<@ NSolve @>, defaultValue = 600) NSpawn = r.GetResult(<@ NSpawn @>, defaultValue = 10) } diff --git a/src/Core/Typedef.fs b/src/Core/Typedef.fs index 1572485..414d935 100644 --- a/src/Core/Typedef.fs +++ b/src/Core/Typedef.fs @@ -24,6 +24,14 @@ type CoverageGain = /// 'Normal' priority. type Priority = Favored | Normal +module Priority = + + let ofCoverageGain = function + | NoGain -> None + | NewPath -> Some Normal + | NewEdge -> Some Favored + + /// Architecture of target program to fuzz. type Arch = X86 | X64 diff --git a/src/Eclipser.fsproj b/src/Eclipser.fsproj index e3a84fa..300a8dd 100644 --- a/src/Eclipser.fsproj +++ b/src/Eclipser.fsproj @@ -25,6 +25,7 @@ + diff --git a/src/Fuzz/Fuzz.fs b/src/Fuzz/Fuzz.fs index a419691..04686f8 100644 --- a/src/Fuzz/Fuzz.fs +++ b/src/Fuzz/Fuzz.fs @@ -10,16 +10,11 @@ let private printFoundSeed verbosity seed = elif verbosity >= 0 then log "[*] Found by grey-box concolic %s" (Seed.toString seed) -let private coverageToPriority = function - | NoGain -> None - | NewPath -> Some Normal - | NewEdge -> Some Favored - let private evalSeed opt seed exitSig covGain = TestCase.save opt seed exitSig covGain if covGain = NewEdge then printFoundSeed opt.Verbosity seed let isAbnormal = Signal.isTimeout exitSig || Signal.isCrash exitSig - if isAbnormal then None else coverageToPriority covGain + if isAbnormal then None else Priority.ofCoverageGain covGain let private initializeSeeds opt = if opt.InputDir = "" then [Seed.make opt.FuzzSource] @@ -28,14 +23,10 @@ let private initializeSeeds opt = |> List.map System.IO.File.ReadAllBytes // Read in file contents |> List.map (Seed.makeWith opt.FuzzSource) // Create seed with content -let private preprocessAux opt seed = +let private makeInitialItems opt seed = let exitSig, covGain = Executor.getCoverage opt seed TestCase.save opt seed exitSig covGain - Option.map (fun pr -> (pr, seed)) (coverageToPriority covGain) - -let private preprocess opt seeds = - log "[*] Total %d initial seeds" (List.length seeds) - List.choose (preprocessAux opt) seeds + Option.map (fun pr -> (pr, seed)) (Priority.ofCoverageGain covGain) let private makeRelocatedItems opt seeds = let collector (seed, exitSig, covGain) = @@ -49,14 +40,21 @@ let private makeSteppedItems pr seed = | None -> [] | Some s -> [(pr, s)] -let rec private fuzzLoop opt seedQueue = +let syncWithAFL opt seedQueue n = + // Sychronize the seed queue with AFL instances every ten iterations. + if n % 10 = 0 && opt.SyncDir <> "" then Sync.run opt seedQueue + else seedQueue + +let rec private fuzzLoop opt seedQueue n = + let verbosity = opt.Verbosity + let seedQueue = syncWithAFL opt seedQueue n if SeedQueue.isEmpty seedQueue then - Thread.Sleep(5000) - fuzzLoop opt seedQueue + if n % 10 = 0 && verbosity >= 1 then log "Seed queue empty, waiting..." + Thread.Sleep(1000) + fuzzLoop opt seedQueue (n + 1) else let priority, seed, seedQueue = SeedQueue.dequeue seedQueue - if opt.Verbosity >= 1 then - log "Grey-box concolic on %A seed : %s" priority (Seed.toString seed) + if verbosity >= 1 then log "Fuzzing with: %s" (Seed.toString seed) let newItems = GreyConcolic.run seed opt // Relocate the cursors of newly generated seeds. let relocatedItems = makeRelocatedItems opt newItems @@ -65,7 +63,7 @@ let rec private fuzzLoop opt seedQueue = // Add the new items to the seed queue. let seedQueue = List.fold SeedQueue.enqueue seedQueue relocatedItems let seedQueue = List.fold SeedQueue.enqueue seedQueue steppedItems - fuzzLoop opt seedQueue + fuzzLoop opt seedQueue (n + 1) let private fuzzingTimer timeoutSec = async { let timespan = System.TimeSpan(0, 0, 0, timeoutSec) @@ -90,9 +88,10 @@ let main args = Executor.initialize opt let emptyQueue = SeedQueue.initialize () let initialSeeds = initializeSeeds opt - let initItems = preprocess opt initialSeeds + log "[*] Total %d initial seeds" (List.length initialSeeds) + let initItems = List.choose (makeInitialItems opt) initialSeeds let initQueue = List.fold SeedQueue.enqueue emptyQueue initItems - log "[*] Fuzzing starts" + log "[*] Start fuzzing" Async.Start (fuzzingTimer opt.Timelimit) - fuzzLoop opt initQueue + fuzzLoop opt initQueue 0 0 // Unreachable diff --git a/src/Fuzz/Sync.fs b/src/Fuzz/Sync.fs new file mode 100644 index 0000000..e712894 --- /dev/null +++ b/src/Fuzz/Sync.fs @@ -0,0 +1,49 @@ +module Eclipser.Sync + +open System.IO +open System.Collections.Generic +open Utils +open Options + +(*** Map of the maximum ID of the already imported test cases. ***) +let private maxImports = new Dictionary () + +let private tryParseTCNum (tcPath: string) = + let tcName = Path.GetFileName(tcPath) + if not(tcName.StartsWith("id:")) then None + else try Some (int (tcName.[3..8])) with _ -> None + +let private importSeed opt tcPath seedQueue = + let tcBytes = File.ReadAllBytes(tcPath) + let seed = Seed.makeWith opt.FuzzSource tcBytes + let covGain = Executor.getCoverage opt seed |> snd + match Priority.ofCoverageGain covGain with + | None -> seedQueue + | Some priority -> SeedQueue.enqueue seedQueue (priority, seed) + +let private syncTestCase opt maxImport (accSeedQueue, accMaxImport) tcPath = + match tryParseTCNum tcPath with + | None -> (accSeedQueue, accMaxImport) + | Some num when num <= maxImport -> (accSeedQueue, accMaxImport) + | Some num -> // Unhandled test case ID. + log "Synchronizing seed queue with %s" tcPath + let accMaxImport = if num > accMaxImport then num else accMaxImport + let accSeedQueue = importSeed opt tcPath accSeedQueue + (accSeedQueue, accMaxImport) + +let private syncFromDir opt seedQueue dir = + let maxImport = if maxImports.ContainsKey(dir) then maxImports.[dir] else 0 + let tcDir = Path.Combine(dir, "queue") + let tcList = Directory.EnumerateFiles(tcDir) |> List.ofSeq + let folder = syncTestCase opt maxImport + let seedQueue, newMaxImport = List.fold folder (seedQueue, maxImport) tcList + if newMaxImport > maxImport then maxImports.[dir] <- newMaxImport + seedQueue + +let run opt seedQueue = + let outDir = Path.GetFullPath(opt.OutDir) + let syncDir = Path.GetFullPath(opt.SyncDir) + let subDirs = Directory.EnumerateDirectories(syncDir) |> List.ofSeq + |> List.map (fun dirName -> Path.Combine(syncDir, dirName)) + |> List.filter (fun d -> d <> outDir) // Exclude our own output. + List.fold (syncFromDir opt) seedQueue subDirs diff --git a/src/Fuzz/TestCase.fs b/src/Fuzz/TestCase.fs index a9a439d..d8ba27f 100644 --- a/src/Fuzz/TestCase.fs +++ b/src/Fuzz/TestCase.fs @@ -47,13 +47,13 @@ let private updateTestcaseCount () = let private dumpCrash opt seed exitSig = if opt.Verbosity >= 0 then log "Save crash seed : %s" (Seed.toString seed) - let crashName = sprintf "crash-%05d" crashCount + let crashName = sprintf "id:%06d" crashCount let crashPath = System.IO.Path.Combine(crashDir, crashName) System.IO.File.WriteAllBytes(crashPath, Seed.concretize seed) updateCrashCount exitSig let private dumpTestCase seed = - let tcName = sprintf "tc-%05d" testCaseCount + let tcName = sprintf "id:%06d" testCaseCount let tcPath = System.IO.Path.Combine(testcaseDir, tcName) System.IO.File.WriteAllBytes(tcPath, Seed.concretize seed) updateTestcaseCount () From 60abfd7d75d728ddb6d9af85e5a185b84fe00965 Mon Sep 17 00:00:00 2001 From: Jaeseung Choi Date: Fri, 25 Sep 2020 01:03:10 -0700 Subject: [PATCH 10/29] Add a toy example to test integration with AFL --- examples/extend.c | 38 ++++++++++++++++++++++++++++++++ examples/test_extend.sh | 49 +++++++++++++++++++++++++++++++++++++++++ 2 files changed, 87 insertions(+) create mode 100644 examples/extend.c create mode 100755 examples/test_extend.sh diff --git a/examples/extend.c b/examples/extend.c new file mode 100644 index 0000000..c5c9e76 --- /dev/null +++ b/examples/extend.c @@ -0,0 +1,38 @@ +/* A simple example to test whether Eclipser can cooperate well with AFL. */ + +#include +#include +#include +#include +#include +#include + +int main(int argc, char ** argv){ + int i; + int fd; + size_t n; + char buf[12]; + + if (argc < 2) + return -1; + + fd = open(argv[1], O_RDWR); + + read(fd, &i, sizeof(int)); + if (i == 0x41424344) { + printf("Found new path 1!\n"); + read(fd, buf, 12); + n = read(fd, &i, sizeof(int)); + // Input length should be greater than 20 byte for this. + if (n == sizeof(int)) { + printf("Found new path 2!\n"); + if (i == 0x61626364) { + abort(); + } + } + } + + close(fd); + + return 0; +} diff --git a/examples/test_extend.sh b/examples/test_extend.sh new file mode 100755 index 0000000..ef7df31 --- /dev/null +++ b/examples/test_extend.sh @@ -0,0 +1,49 @@ +#!/bin/bash + +if [ -z "$1" ] +then + echo "Should provide AFL root directory as argument" + exit 1 +fi + +if [ ! -d $1 ] +then + echo "Cannot find AFL root directory path $1" + exit 1 +fi + +gcc extend.c -o extend.bin -static -g || exit 1 +rm -rf box +mkdir box +cd box +mkdir input +echo "" > input/empty +mkdir afl-box +mkdir eclipser-box +mkdir syncdir + +# Launch master and slave AFLs. +cd afl-box +cp ../../extend.bin ./ +echo "Start master AFL" +CMD="timeout 20 $1/afl-fuzz -i ../input -o ../syncdir/ -M afl-master -Q -- ./extend.bin @@ > log.txt" +echo "#!/bin/bash" > run_master.sh +echo $CMD >> run_master.sh +chmod 755 run_master.sh +./run_master.sh & +echo "Launched master AFL" + +echo "Start slave AFL" +CMD="timeout 20 $1/afl-fuzz -i ../input -o ../syncdir/ -S afl-slave -Q -- ./extend.bin @@ > log.txt" +echo "#!/bin/bash" > run_slave.sh +echo $CMD >> run_slave.sh +chmod 755 run_slave.sh +./run_slave.sh & +echo "Launched slave AFL" + +# Now launch Eclipser. +cd ../eclipser-box +cp ../../extend.bin ./ +dotnet ../../../build/Eclipser.dll \ + -t 20 -v 1 -s ../syncdir -o ../syncdir/eclipser-output \ + -p ./extend.bin --arg input -f input From d91ba30d7abfe8f16164b996b8b97ca6dfa33d9f Mon Sep 17 00:00:00 2001 From: Jaeseung Choi Date: Fri, 25 Sep 2020 02:40:31 -0700 Subject: [PATCH 11/29] Polish QEMU instrumentation code Cleanup memory mapped file when the program under test exits. --- Instrumentor/qemu/patches-branch/chatkey.c | 17 +++++++++++------ Instrumentor/qemu/patches-coverage/chatkey.cc | 17 +++++++++++------ 2 files changed, 22 insertions(+), 12 deletions(-) diff --git a/Instrumentor/qemu/patches-branch/chatkey.c b/Instrumentor/qemu/patches-branch/chatkey.c index a35775a..a6b7acd 100644 --- a/Instrumentor/qemu/patches-branch/chatkey.c +++ b/Instrumentor/qemu/patches-branch/chatkey.c @@ -34,7 +34,7 @@ static char * coverage_path = NULL; static char * branch_path = NULL; static FILE * coverage_fp = NULL; static FILE * branch_fp = NULL; -static unsigned char * accum_edge_bitmap = NULL; +static unsigned char * edge_bitmap = NULL; unsigned char trace_buffer[MAX_TRACE_LEN * (sizeof(abi_ulong) + sizeof(unsigned char) + 2 * sizeof(abi_ulong)) + 64]; unsigned char * buf_ptr = trace_buffer; @@ -50,8 +50,8 @@ void flush_trace_buffer(void) { void chatkey_setup_before_forkserver(void) { char * bitmap_path = getenv("CK_BITMAP_LOG"); int bitmap_fd = open(bitmap_path, O_RDWR | O_CREAT, 0644); - accum_edge_bitmap = (unsigned char*) mmap(NULL, 0x10000, PROT_READ | PROT_WRITE, MAP_SHARED, bitmap_fd, 0); - assert(accum_edge_bitmap != (void *) -1); + edge_bitmap = (unsigned char*) mmap(NULL, 0x10000, PROT_READ | PROT_WRITE, MAP_SHARED, bitmap_fd, 0); + assert(edge_bitmap != (void *) -1); coverage_path = getenv("CK_COVERAGE_LOG"); branch_path = getenv("CK_FEED_LOG"); @@ -117,6 +117,11 @@ void chatkey_exit(void) { fclose(branch_fp); branch_fp = NULL; } + + if (edge_bitmap) { + munmap(edge_bitmap, 0x10000); + edge_bitmap = NULL; + } } /* Recall that in 64bit we already pushed rdi/rsi/rdx before calling @@ -264,7 +269,7 @@ void chatkey_log_bb(abi_ulong addr) { chatkey_curr_addr = addr; - if (!coverage_fp) + if (!coverage_fp || !edge_bitmap) return; #ifdef TARGET_X86_64 @@ -278,10 +283,10 @@ void chatkey_log_bb(abi_ulong addr) { hash = (edge >> 4) ^ (edge << 8); byte_idx = (hash >> 3) & 0xffff; byte_mask = 1 << (hash & 0x7); // Use the lowest 3 bits to shift - old_byte = accum_edge_bitmap[byte_idx]; + old_byte = edge_bitmap[byte_idx]; new_byte = old_byte | byte_mask; if (old_byte != new_byte) { found_new_edge = 1; - accum_edge_bitmap[byte_idx] = new_byte; + edge_bitmap[byte_idx] = new_byte; } } diff --git a/Instrumentor/qemu/patches-coverage/chatkey.cc b/Instrumentor/qemu/patches-coverage/chatkey.cc index 02c91bb..db992f4 100644 --- a/Instrumentor/qemu/patches-coverage/chatkey.cc +++ b/Instrumentor/qemu/patches-coverage/chatkey.cc @@ -26,13 +26,13 @@ static char * coverage_path = NULL; static char * dbg_path = NULL; static FILE * coverage_fp = NULL; static FILE * dbg_fp = NULL; -static unsigned char * accum_edge_bitmap; +static unsigned char * edge_bitmap = NULL; extern "C" void chatkey_setup_before_forkserver(void) { char * bitmap_path = getenv("CK_BITMAP_LOG"); int bitmap_fd = open(bitmap_path, O_RDWR | O_CREAT, 0644); - accum_edge_bitmap = (unsigned char*) mmap(NULL, 0x10000, PROT_READ | PROT_WRITE, MAP_SHARED, bitmap_fd, 0); - assert(accum_edge_bitmap != (void *) -1); + edge_bitmap = (unsigned char*) mmap(NULL, 0x10000, PROT_READ | PROT_WRITE, MAP_SHARED, bitmap_fd, 0); + assert(edge_bitmap != (void *) -1); coverage_path = getenv("CK_COVERAGE_LOG"); dbg_path = getenv("CK_DBG_LOG"); @@ -90,6 +90,11 @@ extern "C" void chatkey_exit(void) { fclose(dbg_fp); dbg_fp = NULL; } + + if (edge_bitmap) { + munmap(edge_bitmap, 0x10000); + edge_bitmap = NULL; + } } extern "C" void chatkey_log_bb(abi_ulong addr) { @@ -97,7 +102,7 @@ extern "C" void chatkey_log_bb(abi_ulong addr) { unsigned int byte_idx, byte_mask; unsigned char old_byte, new_byte; - if (!coverage_fp) + if (!coverage_fp || !edge_bitmap) return; #ifdef TARGET_X86_64 @@ -111,11 +116,11 @@ extern "C" void chatkey_log_bb(abi_ulong addr) { hash = (edge >> 4) ^ (edge << 8); byte_idx = (hash >> 3) & 0xffff; byte_mask = 1 << (hash & 0x7); // Use the lowest 3 bits to shift - old_byte = accum_edge_bitmap[byte_idx]; + old_byte = edge_bitmap[byte_idx]; new_byte = old_byte | byte_mask; if (old_byte != new_byte) { found_new_edge = 1; - accum_edge_bitmap[byte_idx] = new_byte; + edge_bitmap[byte_idx] = new_byte; /* Log visited addrs if dbg_fp is not NULL */ if (dbg_fp) { #ifdef TARGET_X86_64 From d6c83098bf6f44ceb07c7050516dfac9924b0fc0 Mon Sep 17 00:00:00 2001 From: Jaeseung Choi Date: Fri, 25 Sep 2020 03:41:51 -0700 Subject: [PATCH 12/29] Update QEMU to 2.10.0 version - Cleanup instrumentation logics. - Handle TranslationBlock chaining issue. --- .gitignore | 5 +- Instrumentor/qemu/build_qemu_x64.sh | 8 +- Instrumentor/qemu/build_qemu_x86.sh | 8 +- Instrumentor/qemu/gen_bbcount_patch.sh | 22 - Instrumentor/qemu/gen_branch_patch.sh | 32 -- Instrumentor/qemu/gen_coverage_patch.sh | 22 - Instrumentor/qemu/generate_bbcount_patch.sh | 27 ++ Instrumentor/qemu/generate_branch_patch.sh | 59 +++ Instrumentor/qemu/generate_common_patch.sh | 19 + Instrumentor/qemu/generate_coverage_patch.sh | 29 ++ Instrumentor/qemu/patches-bbcount/chatkey.cc | 348 +++----------- .../qemu/patches-bbcount/cpu-exec.diff | 52 +- Instrumentor/qemu/patches-bbcount/main.diff | 139 ------ .../qemu/patches-bbcount/makefile-objs.diff | 9 - .../qemu/patches-bbcount/makefile-target.diff | 21 +- .../qemu/patches-bbcount/syscall.diff | 231 +-------- .../qemu/patches-branch/afl-qemu-cpu-inl.h | 27 +- Instrumentor/qemu/patches-branch/chatkey.c | 36 +- .../qemu/patches-branch/cpu-exec.diff | 63 ++- .../qemu/patches-branch/makefile-target.diff | 18 +- .../qemu/patches-branch/optimize.diff | 93 +--- Instrumentor/qemu/patches-branch/syscall.diff | 259 ++-------- Instrumentor/qemu/patches-branch/tcg-op.diff | 116 ++--- Instrumentor/qemu/patches-branch/tcg-opc.diff | 12 +- .../qemu/patches-branch/tcg-target.diff | 99 +--- .../qemu/patches-branch/translate.diff | 449 +++++------------- .../qemu/patches-common/configure.diff | 43 +- Instrumentor/qemu/patches-common/elfload.diff | 41 +- .../qemu/patches-common/linuxload.diff | 22 - Instrumentor/qemu/patches-common/memfd.diff | 13 + .../qemu/patches-common/os-posix.diff | 13 - Instrumentor/qemu/patches-common/signal.diff | 191 +------- .../qemu/patches-common/texi2pod.diff | 11 - .../qemu/patches-common/translate-all.diff | 18 - .../qemu/patches-common/user-exec.diff | 119 ----- .../qemu/patches-coverage/afl-qemu-cpu-inl.h | 28 +- .../{chatkey.cc => chatkey.c} | 42 +- .../qemu/patches-coverage/cpu-exec.diff | 63 ++- .../qemu/patches-coverage/exec-all.diff | 12 - .../qemu/patches-coverage/makefile-objs.diff | 7 + .../patches-coverage/makefile-target.diff | 20 - .../qemu/patches-coverage/syscall.diff | 259 ++-------- .../qemu/patches-coverage/translate.diff | 219 --------- Instrumentor/qemu/prepare_qemu.sh | 111 ++--- Instrumentor/qemu/rebuild.sh | 84 ---- Instrumentor/qemu/repatch.sh | 184 +++---- Instrumentor/sparsehash/build_sparsehash.sh | 32 -- .../sparsehash/sparsehash-2.0.3.tar.gz.sha512 | 1 - Makefile | 13 +- src/Core/Executor.fs | 2 + 50 files changed, 862 insertions(+), 2889 deletions(-) delete mode 100755 Instrumentor/qemu/gen_bbcount_patch.sh delete mode 100755 Instrumentor/qemu/gen_branch_patch.sh delete mode 100755 Instrumentor/qemu/gen_coverage_patch.sh create mode 100755 Instrumentor/qemu/generate_bbcount_patch.sh create mode 100755 Instrumentor/qemu/generate_branch_patch.sh create mode 100755 Instrumentor/qemu/generate_common_patch.sh create mode 100755 Instrumentor/qemu/generate_coverage_patch.sh delete mode 100644 Instrumentor/qemu/patches-bbcount/main.diff delete mode 100644 Instrumentor/qemu/patches-bbcount/makefile-objs.diff delete mode 100644 Instrumentor/qemu/patches-common/linuxload.diff create mode 100644 Instrumentor/qemu/patches-common/memfd.diff delete mode 100644 Instrumentor/qemu/patches-common/os-posix.diff delete mode 100644 Instrumentor/qemu/patches-common/texi2pod.diff delete mode 100644 Instrumentor/qemu/patches-common/translate-all.diff delete mode 100644 Instrumentor/qemu/patches-common/user-exec.diff rename Instrumentor/qemu/patches-coverage/{chatkey.cc => chatkey.c} (78%) delete mode 100644 Instrumentor/qemu/patches-coverage/exec-all.diff create mode 100644 Instrumentor/qemu/patches-coverage/makefile-objs.diff delete mode 100644 Instrumentor/qemu/patches-coverage/makefile-target.diff delete mode 100644 Instrumentor/qemu/patches-coverage/translate.diff delete mode 100755 Instrumentor/qemu/rebuild.sh delete mode 100755 Instrumentor/sparsehash/build_sparsehash.sh delete mode 100644 Instrumentor/sparsehash/sparsehash-2.0.3.tar.gz.sha512 diff --git a/.gitignore b/.gitignore index 5f2d9ac..c50317b 100644 --- a/.gitignore +++ b/.gitignore @@ -12,11 +12,8 @@ obj/ .vscode/ #etc -Instrumentor/sparsehash/sparsehash-2.0.3 -Instrumentor/sparsehash/sparsehash-2.0.3.tar.gz -Instrumentor/sparsehash/.compiled Instrumentor/qemu/.prepared -Instrumentor/qemu/qemu-2.3.0* +Instrumentor/qemu/qemu-2.10.0* Instrumentor/qemu/.compiled_x86 Instrumentor/qemu/.compiled_x64 Instrumentor/qemu/.compiled diff --git a/Instrumentor/qemu/build_qemu_x64.sh b/Instrumentor/qemu/build_qemu_x64.sh index 97be8f8..73f9f61 100755 --- a/Instrumentor/qemu/build_qemu_x64.sh +++ b/Instrumentor/qemu/build_qemu_x64.sh @@ -16,13 +16,15 @@ # # http://www.apache.org/licenses/LICENSE-2.0 +VERSION="2.10.0" + build_qemu () { echo "[*] Configuring QEMU for x86_64..." - cd qemu-2.3.0-$1-x64 || exit 1 + cd qemu-${VERSION}-$1-x64 || exit 1 CFLAGS="-O3" ./configure --disable-system --enable-linux-user \ - --python=python2 --enable-guest-base --disable-gtk --disable-sdl --disable-vnc \ + --python=python2 --disable-gtk --disable-sdl --disable-vnc \ --target-list="x86_64-linux-user" || exit 1 echo "[+] Configuration complete." @@ -38,8 +40,6 @@ build_qemu () { cd .. } -### Build QEMU tracers - build_qemu coverage mv "./qemu-trace" "../../build/qemu-trace-coverage-x64" || exit 1 echo "[+] Successfully created 'qemu-trace-coverage-x64'." diff --git a/Instrumentor/qemu/build_qemu_x86.sh b/Instrumentor/qemu/build_qemu_x86.sh index 50a57ab..874714c 100755 --- a/Instrumentor/qemu/build_qemu_x86.sh +++ b/Instrumentor/qemu/build_qemu_x86.sh @@ -16,13 +16,15 @@ # # http://www.apache.org/licenses/LICENSE-2.0 +VERSION="2.10.0" + build_qemu () { echo "[*] Configuring QEMU for i386..." - cd qemu-2.3.0-$1-x86 || exit 1 + cd qemu-${VERSION}-$1-x86 || exit 1 CFLAGS="-O3" ./configure --disable-system --enable-linux-user \ - --python=python2 --enable-guest-base --disable-gtk --disable-sdl --disable-vnc \ + --python=python2 --disable-gtk --disable-sdl --disable-vnc \ --target-list="i386-linux-user" || exit 1 echo "[+] Configuration complete." @@ -38,8 +40,6 @@ build_qemu () { cd .. } -### Build QEMU tracers - build_qemu coverage mv "./qemu-trace" "../../build/qemu-trace-coverage-x86" || exit 1 echo "[+] Successfully created 'qemu-trace-coverage-x86'." diff --git a/Instrumentor/qemu/gen_bbcount_patch.sh b/Instrumentor/qemu/gen_bbcount_patch.sh deleted file mode 100755 index 740d2a2..0000000 --- a/Instrumentor/qemu/gen_bbcount_patch.sh +++ /dev/null @@ -1,22 +0,0 @@ -#!/bin/bash - -cp -r qemu-2.3.0-bbcount-x64 qemu-2.3.0-bbcount - -cp qemu-2.3.0-bbcount/chatkey.cc ./patches-bbcount/chatkey.cc - -cp qemu-2.3.0/linux-user/syscall.c qemu-2.3.0-bbcount/linux-user/syscall.c.orig -diff -Naur qemu-2.3.0-bbcount/linux-user/syscall.c.orig qemu-2.3.0-bbcount/linux-user/syscall.c > patches-bbcount/syscall.diff - -cp qemu-2.3.0/cpu-exec.c qemu-2.3.0-bbcount/cpu-exec.c.orig -diff -Naur qemu-2.3.0-bbcount/cpu-exec.c.orig qemu-2.3.0-bbcount/cpu-exec.c > patches-bbcount/cpu-exec.diff - -cp qemu-2.3.0/Makefile.target qemu-2.3.0-bbcount/Makefile.target.orig -diff -Naur qemu-2.3.0-bbcount/Makefile.target.orig qemu-2.3.0-bbcount/Makefile.target > patches-bbcount/makefile-target.diff - -cp qemu-2.3.0/linux-user/Makefile.objs qemu-2.3.0-bbcount/linux-user/Makefile.objs.orig -diff -Naur qemu-2.3.0-bbcount/linux-user/Makefile.objs.orig qemu-2.3.0-bbcount/linux-user/Makefile.objs > patches-bbcount/makefile-objs.diff - -cp qemu-2.3.0/linux-user/main.c qemu-2.3.0-bbcount/linux-user/main.c.orig -diff -Naur qemu-2.3.0-bbcount/linux-user/main.c.orig qemu-2.3.0-bbcount/linux-user/main.c > patches-bbcount/main.diff - -rm -rf qemu-2.3.0 qemu-2.3.0-bbcount diff --git a/Instrumentor/qemu/gen_branch_patch.sh b/Instrumentor/qemu/gen_branch_patch.sh deleted file mode 100755 index 02b9bfb..0000000 --- a/Instrumentor/qemu/gen_branch_patch.sh +++ /dev/null @@ -1,32 +0,0 @@ -#!/bin/bash - -cp qemu-2.3.0-branch/tcg/chatkey.c ./patches-branch/chatkey.c - -cp qemu-2.3.0-branch/afl-qemu-cpu-inl.h ./patches-branch/afl-qemu-cpu-inl.h - -cp qemu-2.3.0/target-i386/translate.c qemu-2.3.0-branch/target-i386/translate.c.orig -diff -Naur qemu-2.3.0-branch/target-i386/translate.c.orig qemu-2.3.0-branch/target-i386/translate.c > patches-branch/translate.diff - -cp qemu-2.3.0/cpu-exec.c qemu-2.3.0-branch/cpu-exec.c.orig -diff -Naur qemu-2.3.0-branch/cpu-exec.c.orig qemu-2.3.0-branch/cpu-exec.c > patches-branch/cpu-exec.diff - -cp qemu-2.3.0/linux-user/syscall.c qemu-2.3.0-branch/linux-user/syscall.c.orig -diff -Naur qemu-2.3.0-branch/linux-user/syscall.c.orig qemu-2.3.0-branch/linux-user/syscall.c > patches-branch/syscall.diff - -cp qemu-2.3.0/Makefile.target qemu-2.3.0-branch/Makefile.target.orig -diff -Naur qemu-2.3.0-branch/Makefile.target.orig qemu-2.3.0-branch/Makefile.target > patches-branch/makefile-target.diff - -cp qemu-2.3.0/tcg/tcg-opc.h qemu-2.3.0-branch/tcg/tcg-opc.h.orig -diff -Naur qemu-2.3.0-branch/tcg/tcg-opc.h.orig qemu-2.3.0-branch/tcg/tcg-opc.h > patches-branch/tcg-opc.diff - -cp qemu-2.3.0/tcg/tcg-op.h qemu-2.3.0-branch/tcg/tcg-op.h.orig -diff -Naur qemu-2.3.0-branch/tcg/tcg-op.h.orig qemu-2.3.0-branch/tcg/tcg-op.h > patches-branch/tcg-op.diff - -cp qemu-2.3.0/tcg/i386/tcg-target.c qemu-2.3.0-branch/tcg/i386/tcg-target.c.orig -diff -Naur qemu-2.3.0-branch/tcg/i386/tcg-target.c.orig qemu-2.3.0-branch/tcg/i386/tcg-target.c > patches-branch/tcg-target.diff - -cp qemu-2.3.0/tcg/tcg.h qemu-2.3.0-branch/tcg/tcg.h.orig -diff -Naur qemu-2.3.0-branch/tcg/tcg.h.orig qemu-2.3.0-branch/tcg/tcg.h > patches-branch/tcg.diff - -cp qemu-2.3.0/tcg/optimize.c qemu-2.3.0-branch/tcg/optimize.c.orig -diff -Naur qemu-2.3.0-branch/tcg/optimize.c.orig qemu-2.3.0-branch/tcg/optimize.c > patches-branch/optimize.diff diff --git a/Instrumentor/qemu/gen_coverage_patch.sh b/Instrumentor/qemu/gen_coverage_patch.sh deleted file mode 100755 index 16bff56..0000000 --- a/Instrumentor/qemu/gen_coverage_patch.sh +++ /dev/null @@ -1,22 +0,0 @@ -#!/bin/bash - -cp qemu-2.3.0-coverage/chatkey.cc ./patches-coverage/chatkey.cc - -cp qemu-2.3.0-coverage/afl-qemu-cpu-inl.h ./patches-coverage/afl-qemu-cpu-inl.h - -cp qemu-2.3.0-coverage/chatkey-utils.h ./patches-coverage/chatkey-utils.h - -cp qemu-2.3.0/linux-user/syscall.c qemu-2.3.0-coverage/linux-user/syscall.c.orig -diff -Naur qemu-2.3.0-coverage/linux-user/syscall.c.orig qemu-2.3.0-coverage/linux-user/syscall.c > patches-coverage/syscall.diff - -cp qemu-2.3.0/target-i386/translate.c qemu-2.3.0-coverage/target-i386/translate.c.orig -diff -Naur qemu-2.3.0-coverage/target-i386/translate.c.orig qemu-2.3.0-coverage/target-i386/translate.c > patches-coverage/translate.diff - -cp qemu-2.3.0/cpu-exec.c qemu-2.3.0-coverage/cpu-exec.c.orig -diff -Naur qemu-2.3.0-coverage/cpu-exec.c.orig qemu-2.3.0-coverage/cpu-exec.c > patches-coverage/cpu-exec.diff - -cp qemu-2.3.0/Makefile.target qemu-2.3.0-coverage/Makefile.target.orig -diff -Naur qemu-2.3.0-coverage/Makefile.target.orig qemu-2.3.0-coverage/Makefile.target > patches-coverage/makefile-target.diff - -cp qemu-2.3.0/include/exec/exec-all.h ./qemu-2.3.0-coverage/include/exec/exec-all.h.orig -diff -Naur qemu-2.3.0-coverage/include/exec/exec-all.h.orig qemu-2.3.0-coverage/include/exec/exec-all.h > patches-coverage/exec-all.diff diff --git a/Instrumentor/qemu/generate_bbcount_patch.sh b/Instrumentor/qemu/generate_bbcount_patch.sh new file mode 100755 index 0000000..a616a64 --- /dev/null +++ b/Instrumentor/qemu/generate_bbcount_patch.sh @@ -0,0 +1,27 @@ +#!/bin/bash + +VERSION="2.10.0" + +cp -r qemu-${VERSION}-bbcount-x64 qemu-${VERSION}-bbcount + +cp qemu-${VERSION}-bbcount/chatkey.cc ./patches-bbcount/ + +cp qemu-${VERSION}/accel/tcg/cpu-exec.c \ + qemu-${VERSION}-bbcount/accel/tcg/cpu-exec.c.orig +diff -Naur qemu-${VERSION}-bbcount/accel/tcg/cpu-exec.c.orig \ + qemu-${VERSION}-bbcount/accel/tcg/cpu-exec.c \ + > patches-bbcount/cpu-exec.diff + +cp qemu-${VERSION}/Makefile.target \ + qemu-${VERSION}-bbcount/Makefile.target.orig +diff -Naur qemu-${VERSION}-bbcount/Makefile.target.orig \ + qemu-${VERSION}-bbcount/Makefile.target \ + > patches-bbcount/makefile-target.diff + +cp qemu-${VERSION}/linux-user/syscall.c \ + qemu-${VERSION}-bbcount/linux-user/syscall.c.orig +diff -Naur qemu-${VERSION}-bbcount/linux-user/syscall.c.orig \ + qemu-${VERSION}-bbcount/linux-user/syscall.c \ + > patches-bbcount/syscall.diff + +rm -rf qemu-${VERSION}-bbcount diff --git a/Instrumentor/qemu/generate_branch_patch.sh b/Instrumentor/qemu/generate_branch_patch.sh new file mode 100755 index 0000000..a6bd677 --- /dev/null +++ b/Instrumentor/qemu/generate_branch_patch.sh @@ -0,0 +1,59 @@ +#!/bin/bash + +VERSION="2.10.0" + +cp -r qemu-${VERSION}-branch-x64 qemu-${VERSION}-branch + +cp qemu-${VERSION}-branch/afl-qemu-cpu-inl.h ./patches-branch/afl-qemu-cpu-inl.h + +cp qemu-${VERSION}-branch/tcg/chatkey.c ./patches-branch/chatkey.c + +cp qemu-${VERSION}/accel/tcg/cpu-exec.c \ + qemu-${VERSION}-branch/accel/tcg/cpu-exec.c.orig +diff -Naur qemu-${VERSION}-branch/accel/tcg/cpu-exec.c.orig \ + qemu-${VERSION}-branch/accel/tcg/cpu-exec.c \ + > patches-branch/cpu-exec.diff + +cp qemu-${VERSION}/Makefile.target \ + qemu-${VERSION}-branch/Makefile.target.orig +diff -Naur qemu-${VERSION}-branch/Makefile.target.orig \ + qemu-${VERSION}-branch/Makefile.target \ + > patches-branch/makefile-target.diff + +cp qemu-${VERSION}/linux-user/syscall.c \ + qemu-${VERSION}-branch/linux-user/syscall.c.orig +diff -Naur qemu-${VERSION}-branch/linux-user/syscall.c.orig \ + qemu-${VERSION}-branch/linux-user/syscall.c \ + > patches-branch/syscall.diff + +cp qemu-${VERSION}/tcg/optimize.c \ + qemu-${VERSION}-branch/tcg/optimize.c.orig +diff -Naur qemu-${VERSION}-branch/tcg/optimize.c.orig \ + qemu-${VERSION}-branch/tcg/optimize.c \ + > patches-branch/optimize.diff + +cp qemu-${VERSION}/tcg/tcg-op.h \ + qemu-${VERSION}-branch/tcg/tcg-op.h.orig +diff -Naur qemu-${VERSION}-branch/tcg/tcg-op.h.orig \ + qemu-${VERSION}-branch/tcg/tcg-op.h \ + > patches-branch/tcg-op.diff + +cp qemu-${VERSION}/tcg/tcg-opc.h \ + qemu-${VERSION}-branch/tcg/tcg-opc.h.orig +diff -Naur qemu-${VERSION}-branch/tcg/tcg-opc.h.orig \ + qemu-${VERSION}-branch/tcg/tcg-opc.h \ + > patches-branch/tcg-opc.diff + +cp qemu-${VERSION}/tcg/i386/tcg-target.inc.c \ + qemu-${VERSION}-branch/tcg/i386/tcg-target.inc.c.orig +diff -Naur qemu-${VERSION}-branch/tcg/i386/tcg-target.inc.c.orig \ + qemu-${VERSION}-branch/tcg/i386/tcg-target.inc.c \ + > patches-branch/tcg-target.diff + +cp qemu-${VERSION}/target/i386/translate.c \ + qemu-${VERSION}-branch/target/i386/translate.c.orig +diff -Naur qemu-${VERSION}-branch/target/i386/translate.c.orig \ + qemu-${VERSION}-branch/target/i386/translate.c \ + > patches-branch/translate.diff + +rm -rf qemu-${VERSION}-branch diff --git a/Instrumentor/qemu/generate_common_patch.sh b/Instrumentor/qemu/generate_common_patch.sh new file mode 100755 index 0000000..1ddd504 --- /dev/null +++ b/Instrumentor/qemu/generate_common_patch.sh @@ -0,0 +1,19 @@ +#!/bin/bash + +VERSION="2.10.0" + +diff -Naur qemu-${VERSION}/configure.orig \ + qemu-${VERSION}/configure \ + > patches-common/configure.diff + +diff -Naur qemu-${VERSION}/linux-user/elfload.c.orig \ + qemu-${VERSION}/linux-user/elfload.c \ + > patches-common/elfload.diff + +diff -Naur qemu-${VERSION}/util/memfd.c.orig \ + qemu-${VERSION}/util/memfd.c \ + > patches-common/memfd.diff + +diff -Naur qemu-${VERSION}/linux-user/signal.c.orig \ + qemu-${VERSION}/linux-user/signal.c \ + > patches-common/signal.diff diff --git a/Instrumentor/qemu/generate_coverage_patch.sh b/Instrumentor/qemu/generate_coverage_patch.sh new file mode 100755 index 0000000..a594e35 --- /dev/null +++ b/Instrumentor/qemu/generate_coverage_patch.sh @@ -0,0 +1,29 @@ +#!/bin/bash + +VERSION="2.10.0" + +cp -r qemu-${VERSION}-coverage-x64 qemu-${VERSION}-coverage + +cp qemu-${VERSION}-coverage/accel/tcg/afl-qemu-cpu-inl.h ./patches-coverage/ + +cp qemu-${VERSION}-coverage/accel/tcg/chatkey.c ./patches-coverage/ + +cp qemu-${VERSION}/accel/tcg/cpu-exec.c \ + qemu-${VERSION}-coverage/accel/tcg/cpu-exec.c.orig +diff -Naur qemu-${VERSION}-coverage/accel/tcg/cpu-exec.c.orig \ + qemu-${VERSION}-coverage/accel/tcg/cpu-exec.c \ + > patches-coverage/cpu-exec.diff + +cp qemu-${VERSION}/accel/tcg/Makefile.objs \ + qemu-${VERSION}-coverage/accel/tcg/Makefile.objs.orig +diff -Naur qemu-${VERSION}-coverage/accel/tcg/Makefile.objs.orig \ + qemu-${VERSION}-coverage/accel/tcg/Makefile.objs \ + > patches-coverage/makefile-objs.diff + +cp qemu-${VERSION}/linux-user/syscall.c \ + qemu-${VERSION}-coverage/linux-user/syscall.c.orig +diff -Naur qemu-${VERSION}-coverage/linux-user/syscall.c.orig \ + qemu-${VERSION}-coverage/linux-user/syscall.c \ + > patches-coverage/syscall.diff + +rm -rf qemu-${VERSION}-coverage diff --git a/Instrumentor/qemu/patches-bbcount/chatkey.cc b/Instrumentor/qemu/patches-bbcount/chatkey.cc index 9ec48a8..1bc5d0f 100644 --- a/Instrumentor/qemu/patches-bbcount/chatkey.cc +++ b/Instrumentor/qemu/patches-bbcount/chatkey.cc @@ -1,214 +1,42 @@ #include -#include -#include +#include +#include #include #include #include -#include -#include - -#include "config.h" - -using google::dense_hash_set; +#include +#include +#include -#ifdef TARGET_X86_64 +using namespace std; -typedef int64_t abi_long; +// It turns out to be hard to import TARGET_X86_64 macro in C++ source, so just +// use uint64_t as abi_ulong, regardless of architecture. typedef uint64_t abi_ulong; -#define TARGET_NR_open 2 -#define TARGET_NR_openat 257 -#define TARGET_NR_mmap 9 -#define TARGET_NR_mmap2 0xdead // do not exist - -#else - -typedef int32_t abi_long; -typedef uint32_t abi_ulong; -#define TARGET_NR_open 5 -#define TARGET_NR_openat 295 -#define TARGET_NR_mmap 90 -#define TARGET_NR_mmap2 192 - -#endif - -typedef struct { - char * filename; - int fd; -} fd_info; - -typedef struct { - abi_ulong start_addr; - abi_ulong end_addr; - abi_ulong lib_hash; - char * libname; // For debugging. -} lib_info; - -#define BUFSIZE 4096 -#define MAX_LIB_CNT 256 -fd_info fd_arr[MAX_LIB_CNT]; -lib_info lib_arr[MAX_LIB_CNT]; -unsigned int lib_cnt = 0; -abi_ulong chatkey_entry_point; /* ELF entry point (_start) */ -unsigned long chatkey_guest_base; +static void init_accum_set(char* path, set *accum_set); +extern "C" void chatkey_setup(void); +extern "C" void chatkey_close_fp(void); +extern "C" void chatkey_exit(void); +extern "C" void chatkey_log_bb(abi_ulong node); -static abi_ulong path_hash = 5381; // djb2 hash +/* ELF entry point (_start). */ +abi_ulong chatkey_entry_point; - -/* Global file pointers */ static FILE* coverage_fp; static FILE* node_fp; static FILE* edge_fp; -static FILE* path_fp; - -static int is_fp_closed = 0; - -// Holds nodes visited until now (accumulative). -static dense_hash_set accum_node_set; -// Holds edges visited until now (accumulative). -static dense_hash_set accum_edge_set; -// Holds path hashes observed until now (accumulative). -static dense_hash_set accum_path_set; -static int prev_node_cnt; -static int prev_edge_cnt; -static int prev_path_cnt; - -/* Used in MODE_COUNTBB */ -static abi_ulong prev_bb = 0; - -/* Functions related to library information logging */ - -/* Calculate the hash value of library name, with djb2 hash */ -static abi_ulong calculate_library_hash(char * libname) { - char * ptr; - abi_ulong hash = 5381; // djb2 hash - for (ptr = libname; *ptr; ptr++) { - hash = ((hash << 5) + hash) + (abi_ulong) (*ptr); - } - return hash; -} - -/* Normalize (i.e. neutralize the randomization) the address of a basic block */ -static abi_ulong normalize_bb(abi_ulong addr) { - int i; - abi_ulong normalized_addr = 0; - - for(i = 0; i < MAX_LIB_CNT && lib_arr[i].lib_hash; i++) { - if (lib_arr[i].start_addr <= addr && addr <= lib_arr[i].end_addr) { - //printf("addr %lx belongs to %s\n", addr, lib_arr[i].libname); - normalized_addr = addr - lib_arr[i].start_addr + lib_arr[i].lib_hash; - return normalized_addr; - } - } - return addr; -} - -static void update_file_descriptors(char * filename, int fd) { - int i; - - // We do not care the cases where open() failed. - if (fd == -1) - return; - - for(i = 0; i < MAX_LIB_CNT; i++) { - - if(fd_arr[i].fd == fd) { - // Overwrite this entry with the new filename - strncpy(fd_arr[i].filename, filename, BUFSIZE); - break; - } - - if(fd_arr[i].filename == NULL) { - // Found the first empty slot. - fd_arr[i].fd = fd; - fd_arr[i].filename = (char*) malloc(BUFSIZE); - strncpy(fd_arr[i].filename, filename, BUFSIZE); - break; - } - } -} - -static void update_library(size_t size, int fd, abi_ulong start_addr) { - int i; - char * libname = NULL; - abi_ulong lib_hash = 0; - abi_ulong end_addr = 0; - - for(i = 0; i < MAX_LIB_CNT && fd_arr[i].filename; i++) { - if(fd_arr[i].fd == fd) { - // Found a matching file descriptor, so retrieve filname. - libname = fd_arr[i].filename; - break; - } - } - if (libname == NULL) - return; - - lib_hash = calculate_library_hash(libname); - end_addr = start_addr + size; - - for(i = 0; i < MAX_LIB_CNT; i++) { - if (lib_arr[i].lib_hash == lib_hash) { - // Duplicate item found. Update address range only if the new mapping - // subsumes the previous mapping. According to our observation on - // executing dynamically linked binary with 'strace', this branch is - // unlikely to be taken at all. - if (start_addr <= lib_arr[i].start_addr && - end_addr >= lib_arr[i].end_addr) - { - printf("Updating the map!\n"); - lib_arr[i].start_addr = start_addr; - lib_arr[i].end_addr = end_addr; - break; - } - - } - - if(lib_arr[i].lib_hash == 0) { - // Found the first empty slot. - lib_arr[i].libname = (char*) malloc(BUFSIZE); - strncpy(lib_arr[i].libname, libname, BUFSIZE); // For debugging - lib_arr[i].lib_hash = lib_hash; - lib_arr[i].start_addr = start_addr; - lib_arr[i].end_addr = end_addr; - break; - } - } -} - -extern "C" void chatkey_post_syscall(int num, abi_long arg1, abi_long arg2, - abi_long arg5, abi_long ret) { - char * addr; - /* Note that we should log and analyze syscall even before chatkey_setup() - * is called, since libraries are opened and mmapped before the entry point - * of a binary is executed. - */ - switch (num) { - case TARGET_NR_open: - addr = (char*) ((abi_ulong) arg1 + chatkey_guest_base); - update_file_descriptors(addr, (int) ret); - break; - case TARGET_NR_openat: - addr = (char*) ((abi_ulong) arg2 + chatkey_guest_base); - update_file_descriptors(addr, (int) ret); - break; - case TARGET_NR_mmap: - case TARGET_NR_mmap2: - update_library((size_t) arg2, (int) arg5, (abi_ulong) ret); - break; - default: - break; - } - - return; -} +static abi_ulong prev_node = 0; +static set accum_node_set; +static set accum_edge_set; +static int last_node_cnt; +static int last_edge_cnt; /* Functions related to basic block tracing */ -static void init_accum_set(char* path, dense_hash_set *accum_set) -{ +static void init_accum_set(char* path, set *accum_set) { int node_fd; off_t node_filesize; static abi_ulong* data; @@ -242,7 +70,6 @@ extern "C" void chatkey_setup(void) { char * coverage_log = getenv("CK_COVERAGE_LOG"); char * node_log = getenv("CK_NODE_LOG"); char * edge_log = getenv("CK_EDGE_LOG"); - char * path_log = getenv("CK_PATH_LOG"); assert(coverage_log != NULL); coverage_fp = fopen(coverage_log, "a"); @@ -254,50 +81,43 @@ extern "C" void chatkey_setup(void) { */ assert(node_log != NULL); assert(edge_log != NULL); - assert(path_log != NULL); node_fp = fopen(node_log, "a+"); edge_fp = fopen(edge_log, "a+"); - path_fp = fopen(path_log, "a+"); assert(node_fp != NULL); assert(edge_fp != NULL); - assert(path_fp != NULL); - - accum_node_set.set_empty_key(0); - accum_edge_set.set_empty_key(0); - accum_path_set.set_empty_key(0); init_accum_set(node_log, &accum_node_set); init_accum_set(edge_log, &accum_edge_set); - init_accum_set(path_log, &accum_path_set); - prev_node_cnt = accum_node_set.size(); - prev_edge_cnt = accum_edge_set.size(); - prev_path_cnt = accum_path_set.size(); + last_node_cnt = accum_node_set.size(); + last_edge_cnt = accum_edge_set.size(); } // When fork() syscall is encountered, child process should call this function extern "C" void chatkey_close_fp(void) { + if (coverage_fp) { + fclose(coverage_fp); + coverage_fp = NULL; + } - is_fp_closed = 1; - - fclose(coverage_fp); - fclose(node_fp); - fclose(edge_fp); - fclose(path_fp); + if (node_fp) { + fclose(node_fp); + node_fp = NULL; + } + if (edge_fp) { + fclose(edge_fp); + edge_fp = NULL; + } } extern "C" void chatkey_exit(void) { sigset_t mask; - unsigned int accum_node_cnt, accum_edge_cnt, accum_path_cnt; - unsigned int new_node_cnt, new_edge_cnt, new_path_cnt; - - // If chatkey_close_fp() was called, then return without any action. - if (is_fp_closed) - return; + unsigned int accum_node_cnt, accum_edge_cnt; + unsigned int new_node_cnt, new_edge_cnt; // Block signals, since we register signal handler that calls chatkey_exit()/ if (sigfillset(&mask) < 0) @@ -305,69 +125,59 @@ extern "C" void chatkey_exit(void) { if (sigprocmask(SIG_BLOCK, &mask, NULL) < 0) return; + accum_node_cnt = accum_node_set.size(); + accum_edge_cnt = accum_edge_set.size(); + new_node_cnt = accum_node_cnt - last_node_cnt; + new_edge_cnt = accum_edge_cnt - last_edge_cnt; + + if(coverage_fp) { + fprintf(coverage_fp, "Visited nodes : %d (+%d)\n", \ + accum_node_cnt, new_node_cnt); + fprintf(coverage_fp, "Visited edges : %d (+%d)\n", \ + accum_edge_cnt, new_edge_cnt); + fprintf(coverage_fp, "=========================\n"); + fclose(coverage_fp); + coverage_fp = NULL; + } - // Add path hash to the set if it's a previously unseen one. - if (accum_path_set.find(path_hash) == accum_path_set.end()) { - accum_path_set.insert(path_hash); - fwrite(&path_hash, sizeof(abi_ulong), 1, path_fp); + if (node_fp) { + fclose(node_fp); + node_fp = NULL; } - accum_node_cnt = accum_node_set.size(); - accum_edge_cnt = accum_edge_set.size(); - accum_path_cnt = accum_path_set.size(); - new_node_cnt = accum_node_cnt - prev_node_cnt; - new_edge_cnt = accum_edge_cnt - prev_edge_cnt; - new_path_cnt = accum_path_cnt - prev_path_cnt; - - fprintf(coverage_fp, "Visited nodes : %d (+%d)\n", \ - accum_node_cnt, new_node_cnt); - fprintf(coverage_fp, "Visited edges : %d (+%d)\n", \ - accum_edge_cnt, new_edge_cnt); - fprintf(coverage_fp, "Explolred paths : %d (+%d)\n", \ - accum_path_cnt, new_path_cnt); - fprintf(coverage_fp, "=========================\n"); - - fclose(coverage_fp); - fclose(node_fp); - fclose(edge_fp); - fclose(path_fp); + if (edge_fp) { + fclose(edge_fp); + edge_fp = NULL; + } } extern "C" void chatkey_log_bb(abi_ulong node) { - abi_ulong edge; - - /* If coverage_fp is NULL, it means that chatkey_setup() is not called yet - * This happens when QEMU is executing a dynamically linked program. - */ - if (!coverage_fp) - return; + abi_ulong edge; - node = normalize_bb(node); + /* If coverage_fp is NULL, it means that chatkey_setup() is not called yet + * This happens when QEMU is executing a dynamically linked program. + */ + if (!coverage_fp || !node_fp || !edge_fp) + return; - /* Hack to convert a pair of BB into a hash value that represents edge */ #ifdef TARGET_X86_64 - edge = (prev_bb << 16) ^ node; + edge = (prev_node << 16) ^ node; #else - edge = (prev_bb << 8) ^ node; + edge = (prev_node << 8) ^ node; #endif + prev_node = node; - /* Note that we dump the set immediately back to file, since we cannot - * guarantee that timeout termination can be handled by signal handler. - */ - if (accum_node_set.find(node) == accum_node_set.end()) { - accum_node_set.insert(node); - fwrite(&node, sizeof(abi_ulong), 1, node_fp); - } - - if (accum_edge_set.find(edge) == accum_edge_set.end()) { - accum_edge_set.insert(edge); - fwrite(&edge, sizeof(abi_ulong), 1, edge_fp); - } - prev_bb = node; -} + /* Note that we dump the set immediately back to file, since we cannot + * guarantee that timeout termination can be handled by signal handler. + */ + if (accum_node_set.find(node) == accum_node_set.end()) { + accum_node_set.insert(node); + fwrite(&node, sizeof(abi_ulong), 1, node_fp); + } + + if (accum_edge_set.find(edge) == accum_edge_set.end()) { + accum_edge_set.insert(edge); + fwrite(&edge, sizeof(abi_ulong), 1, edge_fp); + } -extern "C" void chatkey_update_path_hash(register abi_ulong addr) { - register unsigned int i; - for (i = 0; i < sizeof(abi_ulong); i++) - path_hash = ((path_hash << 5) + path_hash) + ((addr >> (i<<3)) & 0xff); } diff --git a/Instrumentor/qemu/patches-bbcount/cpu-exec.diff b/Instrumentor/qemu/patches-bbcount/cpu-exec.diff index 79a1798..405f109 100644 --- a/Instrumentor/qemu/patches-bbcount/cpu-exec.diff +++ b/Instrumentor/qemu/patches-bbcount/cpu-exec.diff @@ -1,45 +1,25 @@ ---- qemu-2.3.0-bbcount/cpu-exec.c.orig 2018-09-20 11:06:01.755270480 +0900 -+++ qemu-2.3.0-bbcount/cpu-exec.c 2018-09-20 11:04:41.340342516 +0900 -@@ -28,6 +28,11 @@ - #include "exec/memory-internal.h" - #include "qemu/rcu.h" +--- qemu-2.10.0-bbcount/accel/tcg/cpu-exec.c.orig 2020-09-29 08:25:21.151543920 -0700 ++++ qemu-2.10.0-bbcount/accel/tcg/cpu-exec.c 2020-09-29 08:23:51.822938454 -0700 +@@ -36,6 +36,10 @@ + #include "sysemu/cpus.h" + #include "sysemu/replay.h" +extern abi_ulong chatkey_entry_point; +extern void chatkey_setup(void); +extern void chatkey_log_bb(abi_ulong addr); -+extern void chatkey_update_path_hash(abi_ulong addr); + /* -icount align implementation. */ typedef struct SyncClocks { -@@ -492,6 +497,14 @@ - next_tb = 0; - tcg_ctx.tb_ctx.tb_invalidated_flag = 0; - } -+ -+ if(tb->pc == chatkey_entry_point) { -+ chatkey_setup(); -+ } -+ -+ chatkey_log_bb(tb->pc); -+ chatkey_update_path_hash(tb->pc); -+ - if (qemu_loglevel_mask(CPU_LOG_EXEC)) { - qemu_log("Trace %p [" TARGET_FMT_lx "] %s\n", - tb->tc_ptr, tb->pc, lookup_symbol(tb->pc)); -@@ -499,10 +512,11 @@ - /* see if we can patch the calling TB. When the TB - spans two pages, we cannot safely do a direct - jump. */ -- if (next_tb != 0 && tb->page_addr[1] == -1) { -- tb_add_jump((TranslationBlock *)(next_tb & ~TB_EXIT_MASK), -- next_tb & TB_EXIT_MASK, tb); -- } -+ // XXX: chatkey -+ //if (next_tb != 0 && tb->page_addr[1] == -1) { -+ // tb_add_jump((TranslationBlock *)(next_tb & ~TB_EXIT_MASK), -+ // next_tb & TB_EXIT_MASK, tb); -+ //} - have_tb_lock = false; - spin_unlock(&tcg_ctx.tb_ctx.tb_lock); +@@ -144,6 +148,11 @@ + int tb_exit; + uint8_t *tb_ptr = itb->tc_ptr; ++ if(itb->pc == chatkey_entry_point) { ++ chatkey_setup(); ++ } ++ chatkey_log_bb(itb->pc); ++ + qemu_log_mask_and_addr(CPU_LOG_EXEC, itb->pc, + "Trace %p [%d: " TARGET_FMT_lx "] %s\n", + itb->tc_ptr, cpu->cpu_index, itb->pc, diff --git a/Instrumentor/qemu/patches-bbcount/main.diff b/Instrumentor/qemu/patches-bbcount/main.diff deleted file mode 100644 index 2cee7fc..0000000 --- a/Instrumentor/qemu/patches-bbcount/main.diff +++ /dev/null @@ -1,139 +0,0 @@ ---- qemu-2.3.0-bbcount/linux-user/main.c.orig 2018-09-20 16:39:33.113202244 +0900 -+++ qemu-2.3.0-bbcount/linux-user/main.c 2018-09-20 16:39:19.424845328 +0900 -@@ -36,6 +36,8 @@ - - char *exec_path; - -+extern unsigned long chatkey_guest_base; -+ - int singlestep; - const char *filename; - const char *argv0; -@@ -2808,7 +2810,7 @@ - CPUState *cs = CPU(cris_env_get_cpu(env)); - int trapnr, ret; - target_siginfo_t info; -- -+ - while (1) { - cpu_exec_start(cs); - trapnr = cpu_cris_exec (env); -@@ -2828,13 +2830,13 @@ - /* just indicate that signals should be handled asap */ - break; - case EXCP_BREAK: -- ret = do_syscall(env, -- env->regs[9], -- env->regs[10], -- env->regs[11], -- env->regs[12], -- env->regs[13], -- env->pregs[7], -+ ret = do_syscall(env, -+ env->regs[9], -+ env->regs[10], -+ env->regs[11], -+ env->regs[12], -+ env->regs[13], -+ env->pregs[7], - env->pregs[11], - 0, 0); - env->regs[10] = ret; -@@ -2869,7 +2871,7 @@ - CPUState *cs = CPU(mb_env_get_cpu(env)); - int trapnr, ret; - target_siginfo_t info; -- -+ - while (1) { - cpu_exec_start(cs); - trapnr = cpu_mb_exec (env); -@@ -2892,13 +2894,13 @@ - /* Return address is 4 bytes after the call. */ - env->regs[14] += 4; - env->sregs[SR_PC] = env->regs[14]; -- ret = do_syscall(env, -- env->regs[12], -- env->regs[5], -- env->regs[6], -- env->regs[7], -- env->regs[8], -- env->regs[9], -+ ret = do_syscall(env, -+ env->regs[12], -+ env->regs[5], -+ env->regs[6], -+ env->regs[7], -+ env->regs[8], -+ env->regs[9], - env->regs[10], - 0, 0); - env->regs[3] = ret; -@@ -3438,7 +3440,7 @@ - void init_task_state(TaskState *ts) - { - int i; -- -+ - ts->used = 1; - ts->first_free = ts->sigqueue_table; - for (i = 0; i < MAX_SIGQUEUE_SIZE - 1; i++) { -@@ -4083,6 +4085,8 @@ - tcg_prologue_init(&tcg_ctx); - #endif - -+ chatkey_guest_base = guest_base; -+ - #if defined(TARGET_I386) - env->cr[0] = CR0_PG_MASK | CR0_WP_MASK | CR0_PE_MASK; - env->hflags |= HF_PE_MASK | HF_CPL_MASK; -@@ -4300,23 +4304,23 @@ - env->regs[12] = regs->r12; - env->regs[13] = regs->r13; - env->regs[14] = regs->r14; -- env->regs[15] = regs->r15; -- env->regs[16] = regs->r16; -- env->regs[17] = regs->r17; -- env->regs[18] = regs->r18; -- env->regs[19] = regs->r19; -- env->regs[20] = regs->r20; -- env->regs[21] = regs->r21; -- env->regs[22] = regs->r22; -- env->regs[23] = regs->r23; -- env->regs[24] = regs->r24; -- env->regs[25] = regs->r25; -- env->regs[26] = regs->r26; -- env->regs[27] = regs->r27; -- env->regs[28] = regs->r28; -- env->regs[29] = regs->r29; -- env->regs[30] = regs->r30; -- env->regs[31] = regs->r31; -+ env->regs[15] = regs->r15; -+ env->regs[16] = regs->r16; -+ env->regs[17] = regs->r17; -+ env->regs[18] = regs->r18; -+ env->regs[19] = regs->r19; -+ env->regs[20] = regs->r20; -+ env->regs[21] = regs->r21; -+ env->regs[22] = regs->r22; -+ env->regs[23] = regs->r23; -+ env->regs[24] = regs->r24; -+ env->regs[25] = regs->r25; -+ env->regs[26] = regs->r26; -+ env->regs[27] = regs->r27; -+ env->regs[28] = regs->r28; -+ env->regs[29] = regs->r29; -+ env->regs[30] = regs->r30; -+ env->regs[31] = regs->r31; - env->sregs[SR_PC] = regs->pc; - } - #elif defined(TARGET_MIPS) -@@ -4378,7 +4382,7 @@ - env->regs[12] = regs->r12; - env->regs[13] = regs->r13; - env->regs[14] = info->start_stack; -- env->regs[15] = regs->acr; -+ env->regs[15] = regs->acr; - env->pc = regs->erp; - } - #elif defined(TARGET_S390X) diff --git a/Instrumentor/qemu/patches-bbcount/makefile-objs.diff b/Instrumentor/qemu/patches-bbcount/makefile-objs.diff deleted file mode 100644 index 8bf43c3..0000000 --- a/Instrumentor/qemu/patches-bbcount/makefile-objs.diff +++ /dev/null @@ -1,9 +0,0 @@ ---- qemu-2.3.0-bbcount/linux-user/Makefile.objs.orig 2018-05-24 11:30:49.615769240 +0900 -+++ qemu-2.3.0-bbcount/linux-user/Makefile.objs 2018-05-23 17:47:35.165933727 +0900 -@@ -1,5 +1,5 @@ - obj-y = main.o syscall.o strace.o mmap.o signal.o \ -- elfload.o linuxload.o uaccess.o uname.o -+ elfload.o linuxload.o uaccess.o uname.o chatkey.o - - obj-$(TARGET_HAS_BFLT) += flatload.o - obj-$(TARGET_I386) += vm86.o diff --git a/Instrumentor/qemu/patches-bbcount/makefile-target.diff b/Instrumentor/qemu/patches-bbcount/makefile-target.diff index 1c2a629..ad6c8a7 100644 --- a/Instrumentor/qemu/patches-bbcount/makefile-target.diff +++ b/Instrumentor/qemu/patches-bbcount/makefile-target.diff @@ -1,11 +1,10 @@ ---- qemu-2.3.0-bbcount/Makefile.target.orig 2018-05-24 15:42:00.432802634 +0900 -+++ qemu-2.3.0-bbcount/Makefile.target 2018-05-24 15:19:52.472308828 +0900 -@@ -11,7 +11,7 @@ - endif - QEMU_CFLAGS += -I.. -I$(SRC_PATH)/target-$(TARGET_BASE_ARCH) -DNEED_CPU_H - --QEMU_CFLAGS+=-I$(SRC_PATH)/include -+QEMU_CFLAGS+=-I$(SRC_PATH)/include -I$(SRC_PATH)/../../sparsehash/sparsehash-2.0.3/src -I$(SRC_PATH)/../../sparsehash/build/src - - ifdef CONFIG_USER_ONLY - # user emulator name +--- qemu-2.10.0-bbcount/Makefile.target.orig 2020-09-29 08:25:21.155543947 -0700 ++++ qemu-2.10.0-bbcount/Makefile.target 2020-09-29 08:22:26.302363966 -0700 +@@ -92,6 +92,7 @@ + ######################################################### + # cpu emulator library + obj-y += exec.o ++obj-y += chatkey.o + obj-y += accel/ + obj-$(CONFIG_TCG) += tcg/tcg.o tcg/tcg-op.o tcg/optimize.o + obj-$(CONFIG_TCG) += tcg/tcg-common.o tcg/tcg-runtime.o diff --git a/Instrumentor/qemu/patches-bbcount/syscall.diff b/Instrumentor/qemu/patches-bbcount/syscall.diff index 88da0e3..34c9d3f 100644 --- a/Instrumentor/qemu/patches-bbcount/syscall.diff +++ b/Instrumentor/qemu/patches-bbcount/syscall.diff @@ -1,136 +1,24 @@ ---- qemu-2.3.0-bbcount/linux-user/syscall.c.orig 2018-05-24 19:57:25.566247563 +0900 -+++ qemu-2.3.0-bbcount/linux-user/syscall.c 2018-05-24 19:24:26.528658136 +0900 -@@ -115,6 +115,37 @@ +--- qemu-2.10.0-bbcount/linux-user/syscall.c.orig 2020-09-29 08:25:21.159543974 -0700 ++++ qemu-2.10.0-bbcount/linux-user/syscall.c 2020-09-29 08:25:06.959447395 -0700 +@@ -116,6 +116,9 @@ #include "qemu.h" -+extern void chatkey_post_syscall(int num, abi_long arg1, abi_long arg2, -+ abi_long arg5, abi_long ret); // arrg5 for mmap +extern void chatkey_exit(void); +extern void chatkey_close_fp(void); + -+int check_executable(const char* filename); -+ -+int check_executable(const char* filename) { -+ int fd, size, tmp; -+ if (eaccess(filename, R_OK) == 0 && eaccess(filename, X_OK == 0)) { -+ fd = open(filename, O_RDONLY); -+ if (fd < 0) -+ return -1; -+ size = lseek(fd, 0, SEEK_END); -+ lseek(fd, 0, SEEK_SET); -+ if (size < 4) { -+ close(fd); -+ return -1; -+ } -+ if (read(fd, &tmp, 4) != 4) { -+ close(fd); -+ return -1; -+ } -+ close(fd); -+ if (tmp == 0x464c457f) -+ return 0; -+ } -+ -+ return -1; -+} -+ - #define CLONE_NPTL_FLAGS2 (CLONE_SETTLS | \ - CLONE_PARENT_SETTID | CLONE_CHILD_SETTID | CLONE_CHILD_CLEARTID) - -@@ -846,7 +877,7 @@ - { - abi_ulong target_rlim_swap; - rlim_t result; -- -+ - target_rlim_swap = tswapal(target_rlim); - if (target_rlim_swap == TARGET_RLIM_INFINITY) - return RLIM_INFINITY; -@@ -854,7 +885,7 @@ - result = target_rlim_swap; - if (target_rlim_swap != (rlim_t)result) - return RLIM_INFINITY; -- -+ - return result; - } - -@@ -862,13 +893,13 @@ - { - abi_ulong target_rlim_swap; - abi_ulong result; -- -+ - if (rlim == RLIM_INFINITY || rlim != (abi_long)rlim) - target_rlim_swap = TARGET_RLIM_INFINITY; - else - target_rlim_swap = rlim; - result = tswapal(target_rlim_swap); -- -+ - return result; - } - -@@ -1183,9 +1214,9 @@ - abi_ulong target_cmsg_addr; - struct target_cmsghdr *target_cmsg; - socklen_t space = 0; -- -+ - msg_controllen = tswapal(target_msgh->msg_controllen); -- if (msg_controllen < sizeof (struct target_cmsghdr)) -+ if (msg_controllen < sizeof (struct target_cmsghdr)) - goto the_end; - target_cmsg_addr = tswapal(target_msgh->msg_control); - target_cmsg = lock_user(VERIFY_READ, target_cmsg_addr, msg_controllen, 1); -@@ -1255,7 +1286,7 @@ - socklen_t space = 0; - - msg_controllen = tswapal(target_msgh->msg_controllen); -- if (msg_controllen < sizeof (struct target_cmsghdr)) -+ if (msg_controllen < sizeof (struct target_cmsghdr)) - goto the_end; - target_cmsg_addr = tswapal(target_msgh->msg_control); - target_cmsg = lock_user(VERIFY_WRITE, target_cmsg_addr, msg_controllen, 0); -@@ -4293,7 +4324,7 @@ - } - unlock_user_struct(target_ldt_info, ptr, 1); - -- if (ldt_info.entry_number < TARGET_GDT_ENTRY_TLS_MIN || -+ if (ldt_info.entry_number < TARGET_GDT_ENTRY_TLS_MIN || - ldt_info.entry_number > TARGET_GDT_ENTRY_TLS_MAX) - return -TARGET_EINVAL; - seg_32bit = ldt_info.flags & 1; -@@ -4371,7 +4402,7 @@ - lp = (uint32_t *)(gdt_table + idx); - entry_1 = tswap32(lp[0]); - entry_2 = tswap32(lp[1]); -- -+ - read_exec_only = ((entry_2 >> 9) & 1) ^ 1; - contents = (entry_2 >> 10) & 3; - seg_not_present = ((entry_2 >> 15) & 1) ^ 1; -@@ -4387,8 +4418,8 @@ - (read_exec_only << 3) | (limit_in_pages << 4) | - (seg_not_present << 5) | (useable << 6) | (lm << 7); - limit = (entry_1 & 0xffff) | (entry_2 & 0xf0000); -- base_addr = (entry_1 >> 16) | -- (entry_2 & 0xff000000) | -+ base_addr = (entry_1 >> 16) | -+ (entry_2 & 0xff000000) | - ((entry_2 & 0xff) << 16); - target_ldt_info->base_addr = tswapal(base_addr); - target_ldt_info->limit = tswap32(limit); -@@ -4572,6 +4603,7 @@ + #ifndef CLONE_IO + #define CLONE_IO 0x80000000 /* Clone io context */ + #endif +@@ -6354,6 +6357,7 @@ ret = fork(); if (ret == 0) { /* Child Process. */ + chatkey_close_fp(); - rcu_after_fork(); cpu_clone_regs(env, newsp); fork_end(1); -@@ -5561,6 +5593,7 @@ + /* There is a race condition here. The parent process could +@@ -7764,6 +7768,7 @@ #ifdef TARGET_GPROF _mcleanup(); #endif @@ -138,33 +26,7 @@ gdb_exit(cpu_env, arg1); _exit(arg1); ret = 0; /* avoid warning */ -@@ -5749,10 +5782,21 @@ - } - if (!(p = lock_user_string(arg1))) - goto execve_efault; -- ret = get_errno(execve(p, argp, envp)); -- unlock_user(p, arg1, 0); -- -- goto execve_end; -+ /* If execve() call is expected to successfully load and execute the -+ * specified program, call chatkey_exit() and abort, since the newly -+ * executed program is not out target. Also, if the new program is -+ * loaded and executed, it often ignores SIGTERM signal sent from -+ * the driver, and Chatkey falls into infinite loop. */ -+ if (check_executable(p) != 0) { -+ unlock_user(p, arg1, 0); -+ goto execve_efault; -+ } -+ chatkey_exit(); -+ exit(0); -+ //ret = get_errno(execve(p, argp, envp)); -+ //unlock_user(p, arg1, 0); -+ // -+ //goto execve_end; - - execve_efault: - ret = -TARGET_EFAULT; -@@ -7422,6 +7466,7 @@ +@@ -9820,6 +9825,7 @@ #ifdef TARGET_GPROF _mcleanup(); #endif @@ -172,76 +34,3 @@ gdb_exit(cpu_env, arg1); ret = get_errno(exit_group(arg1)); break; -@@ -8316,7 +8361,7 @@ - break; - #if defined(TARGET_NR_fchownat) - case TARGET_NR_fchownat: -- if (!(p = lock_user_string(arg2))) -+ if (!(p = lock_user_string(arg2))) - goto efault; - ret = get_errno(fchownat(arg1, p, low2highuid(arg3), - low2highgid(arg4), arg5)); -@@ -8801,7 +8846,7 @@ - case TARGET_F_GETLK64: - #ifdef TARGET_ARM - if (((CPUARMState *)cpu_env)->eabi) { -- if (!lock_user_struct(VERIFY_READ, target_efl, arg3, 1)) -+ if (!lock_user_struct(VERIFY_READ, target_efl, arg3, 1)) - goto efault; - fl.l_type = tswap16(target_efl->l_type); - fl.l_whence = tswap16(target_efl->l_whence); -@@ -8812,7 +8857,7 @@ - } else - #endif - { -- if (!lock_user_struct(VERIFY_READ, target_fl, arg3, 1)) -+ if (!lock_user_struct(VERIFY_READ, target_fl, arg3, 1)) - goto efault; - fl.l_type = tswap16(target_fl->l_type); - fl.l_whence = tswap16(target_fl->l_whence); -@@ -8825,7 +8870,7 @@ - if (ret == 0) { - #ifdef TARGET_ARM - if (((CPUARMState *)cpu_env)->eabi) { -- if (!lock_user_struct(VERIFY_WRITE, target_efl, arg3, 0)) -+ if (!lock_user_struct(VERIFY_WRITE, target_efl, arg3, 0)) - goto efault; - target_efl->l_type = tswap16(fl.l_type); - target_efl->l_whence = tswap16(fl.l_whence); -@@ -8836,7 +8881,7 @@ - } else - #endif - { -- if (!lock_user_struct(VERIFY_WRITE, target_fl, arg3, 0)) -+ if (!lock_user_struct(VERIFY_WRITE, target_fl, arg3, 0)) - goto efault; - target_fl->l_type = tswap16(fl.l_type); - target_fl->l_whence = tswap16(fl.l_whence); -@@ -8852,7 +8897,7 @@ - case TARGET_F_SETLKW64: - #ifdef TARGET_ARM - if (((CPUARMState *)cpu_env)->eabi) { -- if (!lock_user_struct(VERIFY_READ, target_efl, arg3, 1)) -+ if (!lock_user_struct(VERIFY_READ, target_efl, arg3, 1)) - goto efault; - fl.l_type = tswap16(target_efl->l_type); - fl.l_whence = tswap16(target_efl->l_whence); -@@ -8863,7 +8908,7 @@ - } else - #endif - { -- if (!lock_user_struct(VERIFY_READ, target_fl, arg3, 1)) -+ if (!lock_user_struct(VERIFY_READ, target_fl, arg3, 1)) - goto efault; - fl.l_type = tswap16(target_fl->l_type); - fl.l_whence = tswap16(target_fl->l_whence); -@@ -9819,6 +9864,9 @@ - ret = -TARGET_ENOSYS; - break; - } -+ -+ chatkey_post_syscall(num, arg1, arg2, arg5, ret); -+ - fail: - #ifdef DEBUG - gemu_log(" = " TARGET_ABI_FMT_ld "\n", ret); diff --git a/Instrumentor/qemu/patches-branch/afl-qemu-cpu-inl.h b/Instrumentor/qemu/patches-branch/afl-qemu-cpu-inl.h index 3bbfe48..3a3a44c 100644 --- a/Instrumentor/qemu/patches-branch/afl-qemu-cpu-inl.h +++ b/Instrumentor/qemu/patches-branch/afl-qemu-cpu-inl.h @@ -34,14 +34,11 @@ unsigned int afl_forksrv_pid; /* Function declarations. */ -static void afl_forkserver(CPUArchState*); +static void afl_forkserver(CPUState*); -static void afl_wait_tsl(CPUArchState*, int); +static void afl_wait_tsl(CPUState*, int); static void afl_request_tsl(target_ulong, target_ulong, uint64_t); -static TranslationBlock *tb_find_slow(CPUArchState*, target_ulong, - target_ulong, uint64_t); - /* Data structure passed around by the translate handlers: */ struct afl_tsl { @@ -50,14 +47,14 @@ struct afl_tsl { uint64_t flags; }; - /************************* * ACTUAL IMPLEMENTATION * *************************/ /* Fork server logic, invoked once we hit _start. */ -static void afl_forkserver(CPUArchState *env) { +static void afl_forkserver(CPUState *cpu) { + static unsigned char tmp[4]; uint64_t tmp_input; @@ -113,7 +110,7 @@ static void afl_forkserver(CPUArchState *env) { /* Collect translation requests until child dies and closes the pipe. */ - afl_wait_tsl(env, t_fd[0]); + afl_wait_tsl(cpu, t_fd[0]); /* Get and relay exit status to parent. */ @@ -144,13 +141,13 @@ static void afl_request_tsl(target_ulong pc, target_ulong cb, uint64_t flags) { } - /* This is the other side of the same channel. Since timeouts are handled by afl-fuzz simply killing the child, we can just wait until the pipe breaks. */ -static void afl_wait_tsl(CPUArchState *env, int fd) { +static void afl_wait_tsl(CPUState *cpu, int fd) { struct afl_tsl t; + TranslationBlock *tb; while (1) { @@ -159,7 +156,15 @@ static void afl_wait_tsl(CPUArchState *env, int fd) { if (read(fd, &t, sizeof(struct afl_tsl)) != sizeof(struct afl_tsl)) break; - tb_find_slow(env, t.pc, t.cs_base, t.flags); + tb = tb_htable_lookup(cpu, t.pc, t.cs_base, t.flags); + + if(!tb) { + mmap_lock(); + tb_lock(); + tb_gen_code(cpu, t.pc, t.cs_base, t.flags, 0); + mmap_unlock(); + tb_unlock(); + } } diff --git a/Instrumentor/qemu/patches-branch/chatkey.c b/Instrumentor/qemu/patches-branch/chatkey.c index a6b7acd..52c23c0 100644 --- a/Instrumentor/qemu/patches-branch/chatkey.c +++ b/Instrumentor/qemu/patches-branch/chatkey.c @@ -6,7 +6,11 @@ #include #include #include -#include "tcg.h" +#include "qemu/osdep.h" +#include "qemu-common.h" +#include "exec/cpu-common.h" +#include "tcg/tcg.h" + #ifdef TARGET_X86_64 typedef uint64_t abi_ulong; @@ -20,6 +24,15 @@ extern unsigned int afl_forksrv_pid; #define MAX_TRACE_LEN (1000000) + +void flush_trace_buffer(void); +void chatkey_setup_before_forkserver(void); +void chatkey_setup_after_forkserver(void); +void chatkey_close_fp(void); +void chatkey_exit(void); +void chatkey_log_branch(abi_ulong oprnd1, abi_ulong oprnd2, unsigned char type); +void chatkey_log_bb(abi_ulong addr); + abi_ulong chatkey_entry_point = 0; /* ELF entry point (_start) */ abi_ulong chatkey_curr_addr = 0; abi_ulong chatkey_targ_addr = 0; @@ -29,7 +42,7 @@ int chatkey_EP_passed = 0; static int found_new_edge = 0; static int found_new_path = 0; // TODO. Extend to measure path coverage, too. -static abi_ulong prev_node = 0; +static abi_ulong prev_addr = 0; static char * coverage_path = NULL; static char * branch_path = NULL; static FILE * coverage_fp = NULL; @@ -93,6 +106,11 @@ void chatkey_close_fp(void) { if (afl_forksrv_pid) close(TSL_FD); + + if (edge_bitmap) { + munmap(edge_bitmap, 0x10000); + edge_bitmap = NULL; + } } void chatkey_exit(void) { @@ -188,7 +206,7 @@ void chatkey_log_branch(abi_ulong oprnd1, abi_ulong oprnd2, unsigned char type) } #endif else { - assert(false); + assert(0); } type = compare_type | operand_size; @@ -245,7 +263,7 @@ void chatkey_log_branch(abi_ulong oprnd1, abi_ulong oprnd2, unsigned char type) } #endif else { - assert(false); + assert(0); } type = compare_type | operand_size; @@ -263,21 +281,25 @@ void chatkey_log_branch(abi_ulong oprnd1, abi_ulong oprnd2, unsigned char type) } void chatkey_log_bb(abi_ulong addr) { + abi_ulong prev_addr_local; abi_ulong edge, hash; unsigned int byte_idx, byte_mask; unsigned char old_byte, new_byte; + // Make sure that 'chatkey_curr_addr' and 'prev_addr' are always updated even + // if we just return. chatkey_curr_addr = addr; + prev_addr_local = prev_addr; + prev_addr = addr; if (!coverage_fp || !edge_bitmap) return; #ifdef TARGET_X86_64 - edge = (prev_node << 16) ^ addr; + edge = (prev_addr_local << 16) ^ addr; #else - edge = (prev_node << 8) ^ addr; + edge = (prev_addr_local << 8) ^ addr; #endif - prev_node = addr; // Update bitmap. hash = (edge >> 4) ^ (edge << 8); diff --git a/Instrumentor/qemu/patches-branch/cpu-exec.diff b/Instrumentor/qemu/patches-branch/cpu-exec.diff index 56dd025..fc87757 100644 --- a/Instrumentor/qemu/patches-branch/cpu-exec.diff +++ b/Instrumentor/qemu/patches-branch/cpu-exec.diff @@ -1,44 +1,41 @@ ---- qemu-2.3.0-branch/cpu-exec.c.orig 2017-09-17 01:29:39.966426739 +0900 -+++ qemu-2.3.0-branch/cpu-exec.c 2017-09-17 01:28:53.506606640 +0900 -@@ -28,6 +28,13 @@ - #include "exec/memory-internal.h" - #include "qemu/rcu.h" +--- qemu-2.10.0-branch/accel/tcg/cpu-exec.c.orig 2020-10-03 21:06:01.813430015 -0700 ++++ qemu-2.10.0-branch/accel/tcg/cpu-exec.c 2020-10-03 21:05:57.233402733 -0700 +@@ -36,6 +36,12 @@ + #include "sysemu/cpus.h" + #include "sysemu/replay.h" ++#include "afl-qemu-cpu-inl.h" +extern abi_ulong chatkey_entry_point; /* ELF entry point (_start) */ +extern void chatkey_setup_before_forkserver(void); +extern void chatkey_setup_after_forkserver(void); +extern void chatkey_log_bb(abi_ulong addr); -+ -+#include "afl-qemu-cpu-inl.h" + /* -icount align implementation. */ typedef struct SyncClocks { -@@ -296,8 +303,11 @@ - } - not_found: - /* if no translated code available, then translate it now */ +@@ -143,6 +149,17 @@ + TranslationBlock *last_tb; + int tb_exit; + uint8_t *tb_ptr = itb->tc_ptr; ++ abi_ulong entry_pc; + - tb = tb_gen_code(cpu, pc, cs_base, flags, 0); ++ entry_pc = itb->pc; ++ if(entry_pc == chatkey_entry_point) { ++ chatkey_setup_before_forkserver(); ++ // Resolves util/rcu.c assertion error issue (cf. AFL-2.53b). ++ rcu_disable_atfork(); ++ afl_forkserver(cpu); ++ chatkey_setup_after_forkserver(); ++ } ++ chatkey_log_bb(entry_pc); -+ afl_request_tsl(pc, cs_base, flags); -+ - found: - /* Move the last found TB to the head of the list */ - if (likely(*ptb1)) { -@@ -492,6 +502,15 @@ - next_tb = 0; - tcg_ctx.tb_ctx.tb_invalidated_flag = 0; - } -+ -+ if(tb->pc == chatkey_entry_point) { -+ chatkey_setup_before_forkserver(); -+ afl_forkserver(env); -+ chatkey_setup_after_forkserver(); -+ } -+ -+ chatkey_log_bb(tb->pc); -+ - if (qemu_loglevel_mask(CPU_LOG_EXEC)) { - qemu_log("Trace %p [" TARGET_FMT_lx "] %s\n", - tb->tc_ptr, tb->pc, lookup_symbol(tb->pc)); + qemu_log_mask_and_addr(CPU_LOG_EXEC, itb->pc, + "Trace %p [%d: " TARGET_FMT_lx "] %s\n", +@@ -365,6 +382,7 @@ + if (!tb) { + /* if no translated code available, then translate it now */ + tb = tb_gen_code(cpu, pc, cs_base, flags, 0); ++ afl_request_tsl(pc, cs_base, flags); + } + + mmap_unlock(); diff --git a/Instrumentor/qemu/patches-branch/makefile-target.diff b/Instrumentor/qemu/patches-branch/makefile-target.diff index 87b8e5e..b44f85f 100644 --- a/Instrumentor/qemu/patches-branch/makefile-target.diff +++ b/Instrumentor/qemu/patches-branch/makefile-target.diff @@ -1,11 +1,11 @@ ---- qemu-2.3.0-branch/Makefile.target.orig 2017-04-13 20:44:12.868483773 +0900 -+++ qemu-2.3.0-branch/Makefile.target 2017-04-10 16:46:23.364441824 +0900 -@@ -83,7 +83,7 @@ - ######################################################### - # cpu emulator library - obj-y = exec.o translate-all.o cpu-exec.o --obj-y += tcg/tcg.o tcg/tcg-op.o tcg/optimize.o -+obj-y += tcg/tcg.o tcg/tcg-op.o tcg/optimize.o tcg/chatkey.o - obj-$(CONFIG_TCG_INTERPRETER) += tci.o +--- qemu-2.10.0-branch/Makefile.target.orig 2020-10-03 21:06:01.957430875 -0700 ++++ qemu-2.10.0-branch/Makefile.target 2020-10-03 21:05:57.237402757 -0700 +@@ -94,7 +94,7 @@ + obj-y += exec.o + obj-y += accel/ + obj-$(CONFIG_TCG) += tcg/tcg.o tcg/tcg-op.o tcg/optimize.o +-obj-$(CONFIG_TCG) += tcg/tcg-common.o tcg/tcg-runtime.o ++obj-$(CONFIG_TCG) += tcg/tcg-common.o tcg/tcg-runtime.o tcg/chatkey.o + obj-$(CONFIG_TCG_INTERPRETER) += tcg/tci.o obj-$(CONFIG_TCG_INTERPRETER) += disas/tci.o obj-y += fpu/softfloat.o diff --git a/Instrumentor/qemu/patches-branch/optimize.diff b/Instrumentor/qemu/patches-branch/optimize.diff index e27e70d..c5c9120 100644 --- a/Instrumentor/qemu/patches-branch/optimize.diff +++ b/Instrumentor/qemu/patches-branch/optimize.diff @@ -1,108 +1,45 @@ ---- qemu-2.3.0-branch/tcg/optimize.c.orig 2017-08-22 16:45:31.562156045 +0900 -+++ qemu-2.3.0-branch/tcg/optimize.c 2017-08-22 16:16:49.397284593 +0900 -@@ -258,8 +258,8 @@ - CASE_OP_32_64(add): - return x + y; - -- CASE_OP_32_64(sub): -- return x - y; -+ //CASE_OP_32_64(sub): -+ // return x - y; - - CASE_OP_32_64(mul): - return x * y; -@@ -674,13 +674,19 @@ +--- qemu-2.10.0-branch/tcg/optimize.c.orig 2020-10-03 21:06:01.973430971 -0700 ++++ qemu-2.10.0-branch/tcg/optimize.c 2020-10-03 21:06:00.237420601 -0700 +@@ -678,13 +678,15 @@ continue; } break; -+ /* To implement instrumentation for Chatkey, we disable this -+ * optimization for subtraction. If we substitute '... = $0 - %reg' with -+ * negation, we have to examine 'neg' to see whether we should insert -+ * instrumentation code. This will make our implementation unnecessarily -+ * complicated. -+ * ++ // This case may remove comparison against 0. ++ /* CASE_OP_32_64(sub): { TCGOpcode neg_op; bool have_neg; - if (temps[args[2]].state == TCG_TEMP_CONST) { + if (temp_is_const(args[2])) { - /* Proceed with possible constant folding. */ + / * Proceed with possible constant folding. * / break; } if (opc == INDEX_op_sub_i32) { -@@ -702,6 +708,7 @@ +@@ -705,6 +707,7 @@ } } break; -+ */ ++ */ CASE_OP_32_64(xor): CASE_OP_32_64(nand): - if (temps[args[1]].state != TCG_TEMP_CONST -@@ -763,7 +770,10 @@ + if (!temp_is_const(args[1]) +@@ -762,7 +765,7 @@ /* Simplify expression for "op r, a, const => mov r, a" cases */ switch (opc) { CASE_OP_32_64(add): - CASE_OP_32_64(sub): -+ /* To implement instrumentation for Chatkey, we disable the optimization -+ * that may remove the side effect on eflags we should investigate. -+ */ -+ //CASE_OP_32_64(sub): ++ //CASE_OP_32_64(sub): // This case may remove comparison against 0. CASE_OP_32_64(shl): CASE_OP_32_64(shr): CASE_OP_32_64(sar): -@@ -778,7 +788,10 @@ - goto do_mov3; - } - break; -- CASE_OP_32_64(and): -+ /* To implement instrumentation for Chatkey, we disable the optimization -+ * that may remove the side effect on eflags we should investigate. -+ */ -+ //CASE_OP_32_64(and): - CASE_OP_32_64(orc): - CASE_OP_32_64(eqv): - if (temps[args[1]].state != TCG_TEMP_CONST -@@ -959,7 +972,10 @@ - - /* Simplify expression for "op r, a, 0 => movi r, 0" cases */ - switch (opc) { -- CASE_OP_32_64(and): -+ /* To implement instrumentation for Chatkey, we disable the optimization -+ * that may remove the side effect on eflags we should investigate. -+ */ -+ //CASE_OP_32_64(and): - CASE_OP_32_64(mul): - CASE_OP_32_64(muluh): - CASE_OP_32_64(mulsh): -@@ -976,7 +992,10 @@ +@@ -1003,7 +1006,7 @@ /* Simplify expression for "op r, a, a => mov r, a" cases */ switch (opc) { CASE_OP_32_64(or): - CASE_OP_32_64(and): -+ /* To implement instrumentation for Chatkey, we disable the optimization -+ * that may remove the side effect on eflags we should investigate. -+ */ -+ //CASE_OP_32_64(and): ++ //CASE_OP_32_64(and): // This case may remove 'test r0, r0' comparison. if (temps_are_copies(args[1], args[2])) { - if (temps_are_copies(args[0], args[1])) { - tcg_op_remove(s, op); -@@ -996,7 +1015,7 @@ - /* Simplify expression for "op r, a, a => movi r, 0" cases */ - switch (opc) { - CASE_OP_32_64(andc): -- CASE_OP_32_64(sub): -+ //CASE_OP_32_64(sub): - CASE_OP_32_64(xor): - if (temps_are_copies(args[1], args[2])) { - tcg_opt_gen_movi(s, op, args, opc, args[0], 0); -@@ -1052,7 +1071,7 @@ - goto do_default; - - CASE_OP_32_64(add): -- CASE_OP_32_64(sub): -+ //CASE_OP_32_64(sub): - CASE_OP_32_64(mul): - CASE_OP_32_64(or): - CASE_OP_32_64(and): + tcg_opt_gen_mov(s, op, args, args[0], args[1]); + continue; diff --git a/Instrumentor/qemu/patches-branch/syscall.diff b/Instrumentor/qemu/patches-branch/syscall.diff index ccb0eaa..00b66aa 100644 --- a/Instrumentor/qemu/patches-branch/syscall.diff +++ b/Instrumentor/qemu/patches-branch/syscall.diff @@ -1,156 +1,25 @@ ---- qemu-2.3.0-branch/linux-user/syscall.c.orig 2017-07-20 14:41:00.160771159 +0900 -+++ qemu-2.3.0-branch/linux-user/syscall.c 2017-07-20 14:34:26.127522059 +0900 -@@ -115,6 +115,33 @@ +--- qemu-2.10.0-branch/linux-user/syscall.c.orig 2020-10-03 21:06:01.965430923 -0700 ++++ qemu-2.10.0-branch/linux-user/syscall.c 2020-10-03 21:05:57.281403019 -0700 +@@ -116,6 +116,10 @@ #include "qemu.h" +extern void chatkey_exit(void); +extern void chatkey_close_fp(void); -+int check_executable(const char* filename); -+ -+int check_executable(const char* filename) { -+ int fd, size, tmp; -+ if (eaccess(filename, R_OK) == 0 && eaccess(filename, X_OK == 0)) { -+ fd = open(filename, O_RDONLY); -+ if (fd < 0) -+ return -1; -+ size = lseek(fd, 0, SEEK_END); -+ lseek(fd, 0, SEEK_SET); -+ if (size < 4) { -+ close(fd); -+ return -1; -+ } -+ if (read(fd, &tmp, 4) != 4) { -+ close(fd); -+ return -1; -+ } -+ close(fd); -+ if (tmp == 0x464c457f) -+ return 0; -+ } -+ -+ return -1; -+} - #define CLONE_NPTL_FLAGS2 (CLONE_SETTLS | \ - CLONE_PARENT_SETTID | CLONE_CHILD_SETTID | CLONE_CHILD_CLEARTID) - -@@ -227,7 +254,22 @@ - _syscall3(int,sys_rt_sigqueueinfo,int,pid,int,sig,siginfo_t *,uinfo) - _syscall3(int,sys_syslog,int,type,char*,bufp,int,len) - #if defined(TARGET_NR_tgkill) && defined(__NR_tgkill) --_syscall3(int,sys_tgkill,int,tgid,int,pid,int,sig) -+ +extern unsigned int afl_forksrv_pid; + -+static int sys_tgkill(int tgid, int pid, int sig) { -+ -+ /* Workaround for -lpthread to make abort() work properly, without -+ killing the forkserver due to a prematurely cached PID. */ -+ -+ if (afl_forksrv_pid && afl_forksrv_pid == pid && -+ (sig == SIGABRT || sig == SIGSEGV || sig == SIGFPE || sig == SIGILL)) -+ pid = tgid = getpid(); -+ -+ return syscall(__NR_sys_tgkill, pid, tgid, sig); -+ -+} -+ + #ifndef CLONE_IO + #define CLONE_IO 0x80000000 /* Clone io context */ #endif - #if defined(TARGET_NR_tkill) && defined(__NR_tkill) - _syscall2(int,sys_tkill,int,tid,int,sig) -@@ -846,7 +888,7 @@ - { - abi_ulong target_rlim_swap; - rlim_t result; -- -+ - target_rlim_swap = tswapal(target_rlim); - if (target_rlim_swap == TARGET_RLIM_INFINITY) - return RLIM_INFINITY; -@@ -854,7 +896,7 @@ - result = target_rlim_swap; - if (target_rlim_swap != (rlim_t)result) - return RLIM_INFINITY; -- -+ - return result; - } - -@@ -862,13 +904,13 @@ - { - abi_ulong target_rlim_swap; - abi_ulong result; -- -+ - if (rlim == RLIM_INFINITY || rlim != (abi_long)rlim) - target_rlim_swap = TARGET_RLIM_INFINITY; - else - target_rlim_swap = rlim; - result = tswapal(target_rlim_swap); -- -+ - return result; - } - -@@ -1183,9 +1225,9 @@ - abi_ulong target_cmsg_addr; - struct target_cmsghdr *target_cmsg; - socklen_t space = 0; -- -+ - msg_controllen = tswapal(target_msgh->msg_controllen); -- if (msg_controllen < sizeof (struct target_cmsghdr)) -+ if (msg_controllen < sizeof (struct target_cmsghdr)) - goto the_end; - target_cmsg_addr = tswapal(target_msgh->msg_control); - target_cmsg = lock_user(VERIFY_READ, target_cmsg_addr, msg_controllen, 1); -@@ -1255,7 +1297,7 @@ - socklen_t space = 0; - - msg_controllen = tswapal(target_msgh->msg_controllen); -- if (msg_controllen < sizeof (struct target_cmsghdr)) -+ if (msg_controllen < sizeof (struct target_cmsghdr)) - goto the_end; - target_cmsg_addr = tswapal(target_msgh->msg_control); - target_cmsg = lock_user(VERIFY_WRITE, target_cmsg_addr, msg_controllen, 0); -@@ -4293,7 +4335,7 @@ - } - unlock_user_struct(target_ldt_info, ptr, 1); - -- if (ldt_info.entry_number < TARGET_GDT_ENTRY_TLS_MIN || -+ if (ldt_info.entry_number < TARGET_GDT_ENTRY_TLS_MIN || - ldt_info.entry_number > TARGET_GDT_ENTRY_TLS_MAX) - return -TARGET_EINVAL; - seg_32bit = ldt_info.flags & 1; -@@ -4371,7 +4413,7 @@ - lp = (uint32_t *)(gdt_table + idx); - entry_1 = tswap32(lp[0]); - entry_2 = tswap32(lp[1]); -- -+ - read_exec_only = ((entry_2 >> 9) & 1) ^ 1; - contents = (entry_2 >> 10) & 3; - seg_not_present = ((entry_2 >> 15) & 1) ^ 1; -@@ -4387,8 +4429,8 @@ - (read_exec_only << 3) | (limit_in_pages << 4) | - (seg_not_present << 5) | (useable << 6) | (lm << 7); - limit = (entry_1 & 0xffff) | (entry_2 & 0xf0000); -- base_addr = (entry_1 >> 16) | -- (entry_2 & 0xff000000) | -+ base_addr = (entry_1 >> 16) | -+ (entry_2 & 0xff000000) | - ((entry_2 & 0xff) << 16); - target_ldt_info->base_addr = tswapal(base_addr); - target_ldt_info->limit = tswap32(limit); -@@ -4572,6 +4614,7 @@ +@@ -6354,6 +6358,7 @@ ret = fork(); if (ret == 0) { /* Child Process. */ + chatkey_close_fp(); - rcu_after_fork(); cpu_clone_regs(env, newsp); fork_end(1); -@@ -5561,6 +5604,7 @@ + /* There is a race condition here. The parent process could +@@ -7764,6 +7769,7 @@ #ifdef TARGET_GPROF _mcleanup(); #endif @@ -158,33 +27,7 @@ gdb_exit(cpu_env, arg1); _exit(arg1); ret = 0; /* avoid warning */ -@@ -5749,10 +5793,21 @@ - } - if (!(p = lock_user_string(arg1))) - goto execve_efault; -- ret = get_errno(execve(p, argp, envp)); -- unlock_user(p, arg1, 0); -- -- goto execve_end; -+ /* If execve() call is expected to successfully load and execute the -+ * specified program, call chatkey_exit() and abort, since the newly -+ * executed program is not out target. Also, if the new program is -+ * loaded and executed, it often ignores SIGTERM signal sent from -+ * the driver, and Chatkey falls into infinite loop. */ -+ if (check_executable(p) != 0) { -+ unlock_user(p, arg1, 0); -+ goto execve_efault; -+ } -+ chatkey_exit(); -+ exit(0); -+ //ret = get_errno(execve(p, argp, envp)); -+ //unlock_user(p, arg1, 0); -+ // -+ //goto execve_end; - - execve_efault: - ret = -TARGET_EFAULT; -@@ -7422,6 +7477,7 @@ +@@ -9820,6 +9826,7 @@ #ifdef TARGET_GPROF _mcleanup(); #endif @@ -192,66 +35,26 @@ gdb_exit(cpu_env, arg1); ret = get_errno(exit_group(arg1)); break; -@@ -8316,7 +8372,7 @@ +@@ -11688,8 +11695,20 @@ break; - #if defined(TARGET_NR_fchownat) - case TARGET_NR_fchownat: -- if (!(p = lock_user_string(arg2))) -+ if (!(p = lock_user_string(arg2))) - goto efault; - ret = get_errno(fchownat(arg1, p, low2highuid(arg3), - low2highgid(arg4), arg5)); -@@ -8801,7 +8857,7 @@ - case TARGET_F_GETLK64: - #ifdef TARGET_ARM - if (((CPUARMState *)cpu_env)->eabi) { -- if (!lock_user_struct(VERIFY_READ, target_efl, arg3, 1)) -+ if (!lock_user_struct(VERIFY_READ, target_efl, arg3, 1)) - goto efault; - fl.l_type = tswap16(target_efl->l_type); - fl.l_whence = tswap16(target_efl->l_whence); -@@ -8812,7 +8868,7 @@ - } else - #endif - { -- if (!lock_user_struct(VERIFY_READ, target_fl, arg3, 1)) -+ if (!lock_user_struct(VERIFY_READ, target_fl, arg3, 1)) - goto efault; - fl.l_type = tswap16(target_fl->l_type); - fl.l_whence = tswap16(target_fl->l_whence); -@@ -8825,7 +8881,7 @@ - if (ret == 0) { - #ifdef TARGET_ARM - if (((CPUARMState *)cpu_env)->eabi) { -- if (!lock_user_struct(VERIFY_WRITE, target_efl, arg3, 0)) -+ if (!lock_user_struct(VERIFY_WRITE, target_efl, arg3, 0)) - goto efault; - target_efl->l_type = tswap16(fl.l_type); - target_efl->l_whence = tswap16(fl.l_whence); -@@ -8836,7 +8892,7 @@ - } else - #endif - { -- if (!lock_user_struct(VERIFY_WRITE, target_fl, arg3, 0)) -+ if (!lock_user_struct(VERIFY_WRITE, target_fl, arg3, 0)) - goto efault; - target_fl->l_type = tswap16(fl.l_type); - target_fl->l_whence = tswap16(fl.l_whence); -@@ -8852,7 +8908,7 @@ - case TARGET_F_SETLKW64: - #ifdef TARGET_ARM - if (((CPUARMState *)cpu_env)->eabi) { -- if (!lock_user_struct(VERIFY_READ, target_efl, arg3, 1)) -+ if (!lock_user_struct(VERIFY_READ, target_efl, arg3, 1)) - goto efault; - fl.l_type = tswap16(target_efl->l_type); - fl.l_whence = tswap16(target_efl->l_whence); -@@ -8863,7 +8919,7 @@ - } else - #endif - { -- if (!lock_user_struct(VERIFY_READ, target_fl, arg3, 1)) -+ if (!lock_user_struct(VERIFY_READ, target_fl, arg3, 1)) - goto efault; - fl.l_type = tswap16(target_fl->l_type); - fl.l_whence = tswap16(target_fl->l_whence); + + case TARGET_NR_tgkill: +- ret = get_errno(safe_tgkill((int)arg1, (int)arg2, +- target_to_host_signal(arg3))); ++ { ++ int pid = (int)arg1, ++ tgid = (int)arg2, ++ sig = (int)arg3; ++ ++ /* Thought to be a workaround for -lpthread to make abort() work ++ * properly, without killing the forkserver due to a prematurely ++ * cached PID. */ ++ ++ if (afl_forksrv_pid && afl_forksrv_pid == pid && sig == SIGABRT) ++ pid = tgid = getpid(); ++ ++ ret = get_errno(safe_tgkill(pid, tgid, target_to_host_signal(sig))); ++ } + break; + + #ifdef TARGET_NR_set_robust_list diff --git a/Instrumentor/qemu/patches-branch/tcg-op.diff b/Instrumentor/qemu/patches-branch/tcg-op.diff index 9b9cb74..dbfb0c3 100644 --- a/Instrumentor/qemu/patches-branch/tcg-op.diff +++ b/Instrumentor/qemu/patches-branch/tcg-op.diff @@ -1,15 +1,24 @@ ---- qemu-2.3.0-branch/tcg/tcg-op.h.orig 2017-08-18 10:10:04.060465652 +0900 -+++ qemu-2.3.0-branch/tcg/tcg-op.h 2017-08-18 10:08:03.058197082 +0900 -@@ -381,14 +381,42 @@ +--- qemu-2.10.0-branch/tcg/tcg-op.h.orig 2020-10-03 21:06:01.977430995 -0700 ++++ qemu-2.10.0-branch/tcg/tcg-op.h 2020-10-03 21:06:00.237420601 -0700 +@@ -26,6 +26,11 @@ + #include "exec/helper-proto.h" + #include "exec/helper-gen.h" + ++#define CHATKEY_CMP_EQUALITY 0 ++#define CHATKEY_CMP_SIZE_SIGNED 1 ++#define CHATKEY_CMP_SIZE_UNSIGNED 2 ++#define CHATKEY_IGNORE 3 ++ + /* Basic output routines. Not for general consumption. */ + + void tcg_gen_op1(TCGContext *, TCGOpcode, TCGArg); +@@ -396,14 +401,36 @@ tcg_gen_op3_i32(INDEX_op_add_i32, ret, arg1, arg2); } - -+/* A variant of tcg_gen_sub_i32() for chatkey. Additional argument is passed -+ * to tcg_gen_op5ii_i32(), to indicate whether this instruction must be -+ * instrumented during TCG-->ASM translation (cf. tcg_out_op()). This value -+ * will be updated appropriately when subsequent conditional jump statement -+ * is encountered. Also, 'TCGMem ot' argument was introduced, to store the -+ * operand size information. + ++/* A variant of tcg_gen_sub_i32() for Chatkey. Additional arguments are passed ++ * to tcg_gen_op5ii_i32(), to indicate the type and size of comparison. We first ++ * pass CHATKEY_IGNORE as comparison type, and update it later. + */ +static inline void tcg_gen_sub_i32_chatkey(TCGv_i32 ret, TCGv_i32 arg1, + TCGv_i32 arg2, TCGMemOp ot) @@ -20,41 +29,35 @@ static inline void tcg_gen_sub_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2) { - tcg_gen_op3_i32(INDEX_op_sub_i32, ret, arg1, arg2); -+ /* Use dummy value MO_8 as operand size */ ++ // Fixed for compatibility. Use dummay values as arguments. + tcg_gen_op5ii_i32(INDEX_op_sub_i32, ret, arg1, arg2, CHATKEY_IGNORE, MO_8); +} + -+/* A variant of tcg_gen_and_i32() for chatkey. Additional argument is passed -+ * to tcg_gen_op5ii_i32(), to indicate whether this instruction must be -+ * instrumented during TCG-->ASM translation (cf. tcg_out_op()). This value -+ * will be updated appropriately when subsequent conditional jump statement -+ * is encountered. Also, 'TCGMem ot' argument was introduced, to store the -+ * operand size information. ++/* A variant of tcg_gen_and_i32() for Chatkey. Additional arguments are passed ++ * to tcg_gen_op5ii_i32(), to indicate the type and size of comparison. We first ++ * pass CHATKEY_IGNORE as comparison type, and update it later. + */ +static inline void tcg_gen_and_i32_chatkey(TCGv_i32 ret, TCGv_i32 arg1, + TCGv_i32 arg2, TCGMemOp ot) +{ + tcg_gen_op5ii_i32(INDEX_op_and_i32, ret, arg1, arg2, CHATKEY_IGNORE, ot); } - + static inline void tcg_gen_and_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2) { - tcg_gen_op3_i32(INDEX_op_and_i32, ret, arg1, arg2); -+ /* Use dummy value MO_8 as operand size */ ++ // Fixed for compatibility. Use dummay values as arguments. + tcg_gen_op5ii_i32(INDEX_op_and_i32, ret, arg1, arg2, CHATKEY_IGNORE, MO_8); } - + static inline void tcg_gen_or_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2) -@@ -580,14 +608,42 @@ +@@ -608,14 +635,36 @@ tcg_gen_op3_i64(INDEX_op_add_i64, ret, arg1, arg2); } - -+/* A variant of tcg_gen_sub_i64() for chatkey. Additional argument is passed -+ * to tcg_gen_op5ii_i32(), to indicate whether this instruction must be -+ * instrumented during TCG-->ASM translation (cf. tcg_out_op()). This value -+ * will be updated appropriately when subsequent conditional jump statement -+ * is encountered. Also, 'TCGMem ot' argument was introduced, to store the -+ * operand size information. + ++/* A variant of tcg_gen_sub_i64() for Chatkey. Additional arguments are passed ++ * to tcg_gen_op5ii_i64(), to indicate the type and size of comparison. We first ++ * pass CHATKEY_IGNORE as comparison type, and update it later. + */ +static inline void tcg_gen_sub_i64_chatkey(TCGv_i64 ret, TCGv_i64 arg1, + TCGv_i64 arg2, TCGMemOp ot) @@ -65,64 +68,47 @@ static inline void tcg_gen_sub_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2) { - tcg_gen_op3_i64(INDEX_op_sub_i64, ret, arg1, arg2); -+ /* Use dummy value MO_8 as operand size */ ++ // Fixed for compatibility. Use dummay values as arguments. + tcg_gen_op5ii_i64(INDEX_op_sub_i64, ret, arg1, arg2, CHATKEY_IGNORE, MO_8); +} + -+/* A variant of tcg_gen_and_i64() for chatkey. Additional argument is passed -+ * to tcg_gen_op5ii_i32(), to indicate whether this instruction must be -+ * instrumented during TCG-->ASM translation (cf. tcg_out_op()). This value -+ * will be updated appropriately when subsequent conditional jump statement -+ * is encountered. Also, 'TCGMem ot' argument was introduced, to store the -+ * operand size information. ++/* A variant of tcg_gen_and_i64() for Chatkey. Additional arguments are passed ++ * to tcg_gen_op5ii_i64(), to indicate the type and size of comparison. We first ++ * pass CHATKEY_IGNORE as comparison type, and update it later. + */ +static inline void tcg_gen_and_i64_chatkey(TCGv_i64 ret, TCGv_i64 arg1, + TCGv_i64 arg2, TCGMemOp ot) +{ + tcg_gen_op5ii_i64(INDEX_op_and_i64, ret, arg1, arg2, CHATKEY_IGNORE, ot); } - + static inline void tcg_gen_and_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2) { - tcg_gen_op3_i64(INDEX_op_and_i64, ret, arg1, arg2); -+ /* Use dummy value MO_8 as operand size */ ++ // Fixed for compatibility. Use dummay values as arguments. + tcg_gen_op5ii_i64(INDEX_op_and_i64, ret, arg1, arg2, CHATKEY_IGNORE, MO_8); } - + static inline void tcg_gen_or_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2) -@@ -644,10 +700,22 @@ +@@ -672,8 +721,17 @@ TCGV_HIGH(arg1), TCGV_LOW(arg2), TCGV_HIGH(arg2)); } - -+/* XXX (Chatkey) -+ * We just leave tcg_gen_sub_i64() below as it is, without adding its -+ * tcg_gen_sub_i64_chatkey(). This is because the definition below is never used -+ * as long as we are targeting Intel architecture. This definition is used when -+ * both 'TCG_TARGET_REG_BITS == 32' and 'TARGET_LONG_BITS == 64' are satisfied. -+ * But if you take a look into target-i386/cpu.h and tcg/i386/tcg-target.h, two -+ * conditions cannot be satisfied at the same time. If we want to extend Chatkey -+ * to support other architectures, we should investigate if the architecture -+ * has 32-bit width registers but supports 64-bit subtraction. If so, below code -+ * must be modified as well. + ++/* We leave the following function as it is, since it is unreachable in Intel ++ * architecture. This definition is used when both 'TCG_TARGET_REG_BITS == 32' ++ * and 'TARGET_LONG_BITS == 64' are satisfied, but this is infeasible according ++ * to target/i386/cpu.h and tcg/i386/tcg-target.h files. If we want to extend ++ * Chatkey to support other architectures, we should investigate if the ++ * architecture has 32-bit width registers but supports 64-bit subtraction. If ++ * so, below code must be modified as well. + */ static inline void tcg_gen_sub_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2) { -+ assert(1 == 2); // Should be unreachable for Intel ++ assert(0); tcg_gen_sub2_i32(TCGV_LOW(ret), TCGV_HIGH(ret), TCGV_LOW(arg1), -- TCGV_HIGH(arg1), TCGV_LOW(arg2), TCGV_HIGH(arg2)); -+ TCGV_HIGH(arg1), TCGV_LOW(arg2), TCGV_HIGH(arg2)); + TCGV_HIGH(arg1), TCGV_LOW(arg2), TCGV_HIGH(arg2)); } - - void tcg_gen_discard_i64(TCGv_i64 arg); -@@ -661,6 +729,7 @@ - void tcg_gen_ld32s_i64(TCGv_i64 ret, TCGv_ptr arg2, tcg_target_long offset); - void tcg_gen_ld_i64(TCGv_i64 ret, TCGv_ptr arg2, tcg_target_long offset); - void tcg_gen_st_i64(TCGv_i64 arg1, TCGv_ptr arg2, tcg_target_long offset); -+// (Chatkey) Same for 'and'. Refer to the comment for tcg_gen_sub_i64() above. - void tcg_gen_and_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2); - void tcg_gen_or_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2); - void tcg_gen_xor_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2); -@@ -826,10 +895,12 @@ +@@ -932,10 +990,12 @@ #define tcg_gen_add_tl tcg_gen_add_i64 #define tcg_gen_addi_tl tcg_gen_addi_i64 #define tcg_gen_sub_tl tcg_gen_sub_i64 @@ -135,7 +121,7 @@ #define tcg_gen_andi_tl tcg_gen_andi_i64 #define tcg_gen_or_tl tcg_gen_or_i64 #define tcg_gen_ori_tl tcg_gen_ori_i64 -@@ -904,10 +975,12 @@ +@@ -1030,10 +1090,12 @@ #define tcg_gen_add_tl tcg_gen_add_i32 #define tcg_gen_addi_tl tcg_gen_addi_i32 #define tcg_gen_sub_tl tcg_gen_sub_i32 diff --git a/Instrumentor/qemu/patches-branch/tcg-opc.diff b/Instrumentor/qemu/patches-branch/tcg-opc.diff index f631cb6..01b4ce7 100644 --- a/Instrumentor/qemu/patches-branch/tcg-opc.diff +++ b/Instrumentor/qemu/patches-branch/tcg-opc.diff @@ -1,6 +1,6 @@ ---- qemu-2.3.0-branch/tcg/tcg-opc.h.orig 2017-04-13 20:44:12.868483773 +0900 -+++ qemu-2.3.0-branch/tcg/tcg-opc.h 2017-04-10 16:46:23.372441850 +0900 -@@ -57,7 +57,7 @@ +--- qemu-2.10.0-branch/tcg/tcg-opc.h.orig 2020-10-03 21:06:01.981431019 -0700 ++++ qemu-2.10.0-branch/tcg/tcg-opc.h 2020-10-03 21:06:00.237420601 -0700 +@@ -59,7 +59,7 @@ DEF(st_i32, 0, 2, 1, 0) /* arith */ DEF(add_i32, 1, 2, 0, 0) @@ -9,7 +9,7 @@ DEF(mul_i32, 1, 2, 0, 0) DEF(div_i32, 1, 2, 0, IMPL(TCG_TARGET_HAS_div_i32)) DEF(divu_i32, 1, 2, 0, IMPL(TCG_TARGET_HAS_div_i32)) -@@ -65,7 +65,7 @@ +@@ -67,7 +67,7 @@ DEF(remu_i32, 1, 2, 0, IMPL(TCG_TARGET_HAS_rem_i32)) DEF(div2_i32, 2, 3, 0, IMPL(TCG_TARGET_HAS_div2_i32)) DEF(divu2_i32, 2, 3, 0, IMPL(TCG_TARGET_HAS_div2_i32)) @@ -18,7 +18,7 @@ DEF(or_i32, 1, 2, 0, 0) DEF(xor_i32, 1, 2, 0, 0) /* shifts/rotates */ -@@ -119,7 +119,7 @@ +@@ -126,7 +126,7 @@ DEF(st_i64, 0, 2, 1, IMPL64) /* arith */ DEF(add_i64, 1, 2, 0, IMPL64) @@ -27,7 +27,7 @@ DEF(mul_i64, 1, 2, 0, IMPL64) DEF(div_i64, 1, 2, 0, IMPL64 | IMPL(TCG_TARGET_HAS_div_i64)) DEF(divu_i64, 1, 2, 0, IMPL64 | IMPL(TCG_TARGET_HAS_div_i64)) -@@ -127,7 +127,7 @@ +@@ -134,7 +134,7 @@ DEF(remu_i64, 1, 2, 0, IMPL64 | IMPL(TCG_TARGET_HAS_rem_i64)) DEF(div2_i64, 2, 3, 0, IMPL64 | IMPL(TCG_TARGET_HAS_div2_i64)) DEF(divu2_i64, 2, 3, 0, IMPL64 | IMPL(TCG_TARGET_HAS_div2_i64)) diff --git a/Instrumentor/qemu/patches-branch/tcg-target.diff b/Instrumentor/qemu/patches-branch/tcg-target.diff index c89fdaf..82a054e 100644 --- a/Instrumentor/qemu/patches-branch/tcg-target.diff +++ b/Instrumentor/qemu/patches-branch/tcg-target.diff @@ -1,70 +1,28 @@ ---- qemu-2.3.0-branch/tcg/i386/tcg-target.c.orig 2017-10-01 21:59:22.790565093 +0900 -+++ qemu-2.3.0-branch/tcg/i386/tcg-target.c 2017-10-01 21:57:46.366637219 +0900 -@@ -24,6 +24,8 @@ +--- qemu-2.10.0-branch/tcg/i386/tcg-target.inc.c.orig 2020-10-03 21:06:01.985431043 -0700 ++++ qemu-2.10.0-branch/tcg/i386/tcg-target.inc.c 2020-10-03 21:06:00.241420625 -0700 +@@ -24,6 +24,10 @@ #include "tcg-be-ldst.h" -+extern void chatkey_trampoline(tcg_target_ulong distance, int operand_size); ++extern void chatkey_trampoline(abi_ulong oprnd1, ++ abi_ulong oprnd2, ++ unsigned char type); + - #ifndef NDEBUG + #ifdef CONFIG_DEBUG_TCG static const char * const tcg_target_reg_names[TCG_TARGET_NB_REGS] = { #if TCG_TARGET_REG_BITS == 64 -@@ -93,7 +95,7 @@ - #define TCG_CT_CONST_U32 0x200 - #define TCG_CT_CONST_I32 0x400 - --/* Registers used with L constraint, which are the first argument -+/* Registers used with L constraint, which are the first argument - registers on x86_64, and two random call clobbered registers on - i386. */ - #if TCG_TARGET_REG_BITS == 64 -@@ -301,8 +303,10 @@ - #define P_SIMDF3 0x10000 /* 0xf3 opcode prefix */ - #define P_SIMDF2 0x20000 /* 0xf2 opcode prefix */ - -+#define OPC_ARITH_EbIb (0x80) - #define OPC_ARITH_EvIz (0x81) - #define OPC_ARITH_EvIb (0x83) -+#define OPC_ARITH_GbEb (0x02) /* ... plus (ARITH_FOO << 3) */ - #define OPC_ARITH_GvEv (0x03) /* ... plus (ARITH_FOO << 3) */ - #define OPC_ANDN (0xf2 | P_EXT38) - #define OPC_ADD_GvEv (OPC_ARITH_GvEv | (ARITH_ADD << 3)) -@@ -832,6 +836,9 @@ - } - - if (val == (int8_t)val) { -+ /* Note that OPC_ARITH_EvIb(0x83) specifies to extend byte operand into -+ * word or dword and then subtract. -+ */ - tcg_out_modrm(s, OPC_ARITH_EvIb + rexw, c, r0); - tcg_out8(s, val); - return; -@@ -842,6 +849,10 @@ - return; - } - -+ /* No handling for 64bit 'val', since 64bit constant cannot exist in both -+ * x86/x64 architecture. -+ */ -+ - tcg_abort(); - } - -@@ -1718,6 +1729,7 @@ - const TCGArg *args, const int *const_args) +@@ -1853,6 +1857,7 @@ { - int c, vexop, rexw = 0; + TCGArg a0, a1, a2; + int c, const_a2, vexop, rexw = 0; + TCGMemOp ot; #if TCG_TARGET_REG_BITS == 64 # define OP_32_64(x) \ -@@ -1822,12 +1834,92 @@ - } +@@ -1976,9 +1981,72 @@ c = ARITH_ADD; goto gen_arith; -+ OP_32_64(sub): -+ + if (args[3] != CHATKEY_IGNORE) { + ot = args[4]; +#if TCG_TARGET_REG_BITS == 64 @@ -72,7 +30,7 @@ + tcg_out_push(s, TCG_REG_RSI); + tcg_out_push(s, TCG_REG_RDX); + tcg_out_mov(s, TCG_TYPE_I64, TCG_REG_RDI, args[0]); -+ if (const_args[2]) { ++ if (const_a2) { + tcg_out_movi(s, TCG_TYPE_I64, TCG_REG_RSI, args[2]); + } else { + tcg_out_mov(s, TCG_TYPE_I64, TCG_REG_RSI, args[2]); @@ -87,7 +45,7 @@ + tcg_out_push(s, TCG_REG_ESI); + tcg_out_push(s, TCG_REG_EDX); + tcg_out_mov(s, TCG_TYPE_I32, TCG_REG_EDI, args[0]); -+ if (const_args[2]) { ++ if (const_a2) { + tcg_out_movi(s, TCG_TYPE_I32, TCG_REG_ESI, args[2]); + } else { + tcg_out_mov(s, TCG_TYPE_I32, TCG_REG_ESI, args[2]); @@ -99,21 +57,13 @@ + tcg_out_pop(s, TCG_REG_EDI); +#endif + } -+ c = ARITH_SUB; -- goto gen_arith; -+ /* Copied from 'gen_arith:' below. */ -+ if (const_args[2]) { -+ tgen_arithi(s, c + rexw, args[0], args[2], 0); -+ } else { -+ tgen_arithr(s, c + rexw, args[0], args[2]); -+ } -+ break; -+ + goto gen_arith; OP_32_64(and): -+ -+ /* Recall that the target of Chatkey is 'test' with same register */ -+ if (args[3] != CHATKEY_IGNORE && !const_args[2]) { ++ /* Recall that we only marked 'test r1, r1' case as our instrumentation ++ * target, so consider it as 'cmp r1, 0'. ++ */ ++ if (args[3] != CHATKEY_IGNORE) { + ot = args[4]; +#if TCG_TARGET_REG_BITS == 64 + tcg_out_push(s, TCG_REG_RDI); @@ -139,17 +89,6 @@ + tcg_out_pop(s, TCG_REG_EDI); +#endif + } -+ c = ARITH_AND; -- goto gen_arith; -+ /* Copied from 'gen_arith:' below. */ -+ if (const_args[2]) { // Copied from 'gen_arith:' below. -+ tgen_arithi(s, c + rexw, args[0], args[2], 0); -+ } else { -+ tgen_arithr(s, c + rexw, args[0], args[2]); -+ } -+ break; -+ - OP_32_64(or): - c = ARITH_OR; goto gen_arith; + OP_32_64(or): diff --git a/Instrumentor/qemu/patches-branch/translate.diff b/Instrumentor/qemu/patches-branch/translate.diff index 9110c50..8854916 100644 --- a/Instrumentor/qemu/patches-branch/translate.diff +++ b/Instrumentor/qemu/patches-branch/translate.diff @@ -1,53 +1,40 @@ ---- qemu-2.3.0-branch/target-i386/translate.c.orig 2017-09-17 00:03:03.803748021 +0900 -+++ qemu-2.3.0-branch/target-i386/translate.c 2017-09-16 23:51:44.773721311 +0900 -@@ -60,6 +60,13 @@ - # define clztl clz32 - #endif +--- qemu-2.10.0-branch/target/i386/translate.c.orig 2020-10-03 21:06:01.989431067 -0700 ++++ qemu-2.10.0-branch/target/i386/translate.c 2020-10-03 21:05:57.213402615 -0700 +@@ -71,6 +71,10 @@ + + //#define MACRO_TEST 1 -+/* A variable introduced in chatkey.c, to decide whether we should start -+ * instrumentation. -+ */ +extern int chatkey_EP_passed; +extern abi_ulong chatkey_curr_addr; +extern abi_ulong chatkey_targ_addr; + - //#define MACRO_TEST 1 - /* global register indexes */ -@@ -126,6 +133,11 @@ - int cpuid_ext2_features; + static TCGv_env cpu_env; + static TCGv cpu_A0; +@@ -138,6 +142,10 @@ int cpuid_ext3_features; int cpuid_7_0_ebx_features; -+ /* Additional information for Chatkey */ -+ int latest_tgt_parm_idx; -+ /* The parameter index of the latest cmp instruction. This index will be -+ * used to update parameter when subsequent jnz instruction is encountered. + int cpuid_xsave_features; ++ /* Additional field for Chatkey, to record the parameter index of the latest ++ * cmp/test instruction. + */ ++ int latest_tgt_parm_idx; } DisasContext; static void gen_eob(DisasContext *s); -@@ -391,7 +403,7 @@ - tcg_gen_addi_tl(cpu_A0, cpu_A0, val); - } - #endif -- -+ - static void gen_add_A0_im(DisasContext *s, int val) - { - #ifdef TARGET_X86_64 -@@ -730,6 +742,11 @@ - tcg_gen_mov_tl(cpu_cc_dst, cpu_T[0]); +@@ -664,6 +672,11 @@ + tcg_gen_mov_tl(cpu_cc_dst, cpu_T0); } +static inline void gen_op_testl_T0_T1_cc_chatkey(TCGMemOp ot) +{ -+ tcg_gen_and_tl_chatkey(cpu_cc_dst, cpu_T[0], cpu_T[1], ot); ++ tcg_gen_and_tl_chatkey(cpu_cc_dst, cpu_T0, cpu_T1, ot); +} + static inline void gen_op_testl_T0_T1_cc(void) { - tcg_gen_and_tl(cpu_cc_dst, cpu_T[0], cpu_T[1]); -@@ -945,7 +962,8 @@ + tcg_gen_and_tl(cpu_cc_dst, cpu_T0, cpu_T1); +@@ -885,7 +898,8 @@ /* perform a conditional store into register 'reg' according to jump opcode value 'b'. In the fast case, T0 is guaranted not to be used. */ @@ -57,7 +44,7 @@ { int inv, jcc_op, cond; TCGMemOp size; -@@ -957,6 +975,8 @@ +@@ -897,6 +911,8 @@ switch (s->cc_op) { case CC_OP_SUBB ... CC_OP_SUBQ: @@ -66,7 +53,7 @@ /* We optimize relational operators for the cmp/jcc case. */ size = s->cc_op - CC_OP_SUBB; switch (jcc_op) { -@@ -1041,9 +1061,9 @@ +@@ -981,9 +997,9 @@ return cc; } @@ -78,7 +65,7 @@ if (cc.no_setcond) { if (cc.cond == TCG_COND_EQ) { -@@ -1073,14 +1093,14 @@ +@@ -1013,14 +1029,14 @@ static inline void gen_compute_eflags_c(DisasContext *s, TCGv reg) { @@ -90,167 +77,126 @@ value 'b'. In the fast case, T0 is guaranted not to be used. */ static inline void gen_jcc1_noeob(DisasContext *s, int b, TCGLabel *l1) { -- CCPrepare cc = gen_prepare_cc(s, b, cpu_T[0]); -+ CCPrepare cc = gen_prepare_cc(s, b, cpu_T[0], 1); +- CCPrepare cc = gen_prepare_cc(s, b, cpu_T0); ++ CCPrepare cc = gen_prepare_cc(s, b, cpu_T0, 1); if (cc.mask != -1) { - tcg_gen_andi_tl(cpu_T[0], cc.reg, cc.mask); -@@ -1098,7 +1118,28 @@ + tcg_gen_andi_tl(cpu_T0, cc.reg, cc.mask); +@@ -1038,7 +1054,26 @@ A translation block must end soon. */ static inline void gen_jcc1(DisasContext *s, int b, TCGLabel *l1) { -- CCPrepare cc = gen_prepare_cc(s, b, cpu_T[0]); +- CCPrepare cc = gen_prepare_cc(s, b, cpu_T0); + CCPrepare cc; + int jcc_op = (b >> 1) & 7; + int cc_optimize_flag = 1; + -+ /* Note that we should examine all the callsites of gen_jcc1() to understand -+ * which branch instructions are instrumentation target of Chatkey. ++ /* If this is a JCC instruction followed by sub/cmp/test, replace the ++ * previous comparison type 'CHATKEY_IGNORE' with a new one, to indicate ++ * that this is our instrumentation target. + */ -+ // TODO : In case of 'test' operation, 'js' instruction may follow to -+ // determine whether the operand is positive or negative. It would be -+ // good to handle this case, too. + if (jcc_op == JCC_Z && s->latest_tgt_parm_idx != -1) { + tcg_ctx.gen_opparam_buf[s->latest_tgt_parm_idx] = CHATKEY_CMP_EQUALITY; -+ cc_optimize_flag = 0; // Disable cc optimization ++ cc_optimize_flag = 0; + } else if ((jcc_op == JCC_B || jcc_op == JCC_BE) && s->latest_tgt_parm_idx != -1) { + tcg_ctx.gen_opparam_buf[s->latest_tgt_parm_idx] = CHATKEY_CMP_SIZE_UNSIGNED; -+ cc_optimize_flag = 0; // Disable cc optimization ++ cc_optimize_flag = 0; + } else if ((jcc_op == JCC_L || jcc_op == JCC_LE) && s->latest_tgt_parm_idx != -1) { + tcg_ctx.gen_opparam_buf[s->latest_tgt_parm_idx] = CHATKEY_CMP_SIZE_SIGNED; -+ cc_optimize_flag = 0; // Disable cc optimization ++ cc_optimize_flag = 0; + } + -+ cc = gen_prepare_cc(s, b, cpu_T[0], cc_optimize_flag); ++ cc = gen_prepare_cc(s, b, cpu_T0, cc_optimize_flag); gen_update_cc_op(s); if (cc.mask != -1) { -@@ -1235,7 +1276,7 @@ - gen_ ## op(s, ot); \ - gen_op_add_reg_im(s->aflag, R_ECX, -1); \ - gen_update_cc_op(s); \ -- gen_jcc1(s, (JCC_Z << 1) | (nz ^ 1), l2); \ -+ gen_jcc1(s, (JCC_Z << 1) | (nz ^ 1), l2); \ - if (s->repz_opt) \ - gen_op_jz_ecx(s->aflag, l2); \ - gen_jmp(s, cur_eip); \ -@@ -1321,6 +1362,7 @@ - gen_op_st_rm_T0_A0(s1, ot, d); +@@ -1281,6 +1316,8 @@ + } gen_op_update3_cc(cpu_tmp4); set_cc_op(s1, CC_OP_ADCB + ot); -+ s1->latest_tgt_parm_idx = -1; // reset ++ // Met an instruction that affects JCC, so disable instrumentation. ++ s1->latest_tgt_parm_idx = -1; break; case OP_SBBL: gen_compute_eflags_c(s1, cpu_tmp4); -@@ -1329,19 +1371,32 @@ - gen_op_st_rm_T0_A0(s1, ot, d); +@@ -1296,6 +1333,8 @@ + } gen_op_update3_cc(cpu_tmp4); set_cc_op(s1, CC_OP_SBBB + ot); -+ s1->latest_tgt_parm_idx = -1; // reset ++ // Met an instruction that affects JCC, so disable instrumentation. ++ s1->latest_tgt_parm_idx = -1; break; case OP_ADDL: - tcg_gen_add_tl(cpu_T[0], cpu_T[0], cpu_T[1]); - gen_op_st_rm_T0_A0(s1, ot, d); + if (s1->prefix & PREFIX_LOCK) { +@@ -1307,16 +1346,26 @@ + } gen_op_update2_cc(); set_cc_op(s1, CC_OP_ADDB + ot); -+ s1->latest_tgt_parm_idx = -1; // reset ++ // Met an instruction that affects JCC, so disable instrumentation. ++ s1->latest_tgt_parm_idx = -1; break; case OP_SUBL: - tcg_gen_mov_tl(cpu_cc_srcT, cpu_T[0]); -- tcg_gen_sub_tl(cpu_T[0], cpu_T[0], cpu_T[1]); -- gen_op_st_rm_T0_A0(s1, ot, d); -- gen_op_update2_cc(); -- set_cc_op(s1, CC_OP_SUBB + ot); -+ -+ if (chatkey_EP_passed) { -+ tcg_gen_sub_tl_chatkey(cpu_T[0], cpu_T[0], cpu_T[1], ot); -+ /* Update the parameter index of the latest cmp/test instruction. */ -+ assert(tcg_ctx.gen_next_parm_idx >= 2); -+ s1->latest_tgt_parm_idx = tcg_ctx.gen_next_parm_idx - 2; -+ gen_op_st_rm_T0_A0(s1, ot, d); -+ gen_op_update2_cc(); -+ set_cc_op(s1, CC_OP_SUBB + ot); -+ } else { -+ tcg_gen_sub_tl(cpu_T[0], cpu_T[0], cpu_T[1]); -+ gen_op_st_rm_T0_A0(s1, ot, d); -+ gen_op_update2_cc(); -+ set_cc_op(s1, CC_OP_SUBB + ot); -+ } - break; - default: - case OP_ANDL: -@@ -1349,24 +1404,42 @@ - gen_op_st_rm_T0_A0(s1, ot, d); + if (s1->prefix & PREFIX_LOCK) { + tcg_gen_neg_tl(cpu_T0, cpu_T1); + tcg_gen_atomic_fetch_add_tl(cpu_cc_srcT, cpu_A0, cpu_T0, + s1->mem_index, ot | MO_LE); ++ // TODO: Try instrumenting this case, too. + tcg_gen_sub_tl(cpu_T0, cpu_cc_srcT, cpu_T1); + } else { + tcg_gen_mov_tl(cpu_cc_srcT, cpu_T0); +- tcg_gen_sub_tl(cpu_T0, cpu_T0, cpu_T1); ++ if (chatkey_EP_passed) { ++ tcg_gen_sub_tl_chatkey(cpu_T0, cpu_T0, cpu_T1, ot); ++ // Record the index of comparison type argument. ++ assert(tcg_ctx.gen_next_parm_idx >= 2); ++ s1->latest_tgt_parm_idx = tcg_ctx.gen_next_parm_idx - 2; ++ } else { ++ tcg_gen_sub_tl(cpu_T0, cpu_T0, cpu_T1); ++ } + gen_op_st_rm_T0_A0(s1, ot, d); + } + gen_op_update2_cc(); +@@ -1333,6 +1382,8 @@ + } gen_op_update1_cc(); set_cc_op(s1, CC_OP_LOGICB + ot); -+ s1->latest_tgt_parm_idx = -1; // reset ++ // Met an instruction that affects JCC, so disable instrumentation. ++ s1->latest_tgt_parm_idx = -1; break; case OP_ORL: - tcg_gen_or_tl(cpu_T[0], cpu_T[0], cpu_T[1]); - gen_op_st_rm_T0_A0(s1, ot, d); + if (s1->prefix & PREFIX_LOCK) { +@@ -1344,6 +1395,8 @@ + } gen_op_update1_cc(); set_cc_op(s1, CC_OP_LOGICB + ot); -+ s1->latest_tgt_parm_idx = -1; // reset ++ // Met an instruction that affects JCC, so disable instrumentation. ++ s1->latest_tgt_parm_idx = -1; break; case OP_XORL: - tcg_gen_xor_tl(cpu_T[0], cpu_T[0], cpu_T[1]); - gen_op_st_rm_T0_A0(s1, ot, d); + if (s1->prefix & PREFIX_LOCK) { +@@ -1355,11 +1408,20 @@ + } gen_op_update1_cc(); set_cc_op(s1, CC_OP_LOGICB + ot); -+ s1->latest_tgt_parm_idx = -1; // reset ++ // Met an instruction that affects JCC, so disable instrumentation. ++ s1->latest_tgt_parm_idx = -1; break; case OP_CMPL: - tcg_gen_mov_tl(cpu_cc_src, cpu_T[1]); - tcg_gen_mov_tl(cpu_cc_srcT, cpu_T[0]); -- tcg_gen_sub_tl(cpu_cc_dst, cpu_T[0], cpu_T[1]); -- set_cc_op(s1, CC_OP_SUBB + ot); -+ /* We will instrument subtract operation generated for OP_CMPL operation -+ * only. Note that we should examine all the callsites of gen_op() to -+ * understand which instruction is instrumented and which are not. For -+ * example, gen_op_jz_ecx() called for translation of 'rep' does not -+ * call gen_op(). Meanwhile, gen_scas()/gen_cmps() internally calls -+ * gen_op(..., OP_CMPL, ...) for string comparison. -+ */ + tcg_gen_mov_tl(cpu_cc_src, cpu_T1); + tcg_gen_mov_tl(cpu_cc_srcT, cpu_T0); +- tcg_gen_sub_tl(cpu_cc_dst, cpu_T0, cpu_T1); + if (chatkey_EP_passed) { -+ tcg_gen_sub_tl_chatkey(cpu_cc_dst, cpu_T[0], cpu_T[1], ot); -+ /* Update the parameter index of the latest cmp/test instruction. */ -+ assert(tcg_ctx.gen_next_parm_idx >= 2); -+ s1->latest_tgt_parm_idx = tcg_ctx.gen_next_parm_idx - 2; -+ set_cc_op(s1, CC_OP_SUBB + ot); ++ tcg_gen_sub_tl_chatkey(cpu_cc_dst, cpu_T0, cpu_T1, ot); ++ // Record the index of comparison type argument. ++ assert(tcg_ctx.gen_next_parm_idx >= 2); ++ s1->latest_tgt_parm_idx = tcg_ctx.gen_next_parm_idx - 2; + } else { -+ tcg_gen_sub_tl(cpu_cc_dst, cpu_T[0], cpu_T[1]); -+ set_cc_op(s1, CC_OP_SUBB + ot); ++ tcg_gen_sub_tl(cpu_cc_dst, cpu_T0, cpu_T1); + } + set_cc_op(s1, CC_OP_SUBB + ot); break; } - } -@@ -1586,14 +1659,14 @@ - t0 = tcg_const_i32(0); - t1 = tcg_temp_new_i32(); - tcg_gen_trunc_tl_i32(t1, cpu_T[1]); -- tcg_gen_movi_i32(cpu_tmp2_i32, CC_OP_ADCOX); -+ tcg_gen_movi_i32(cpu_tmp2_i32, CC_OP_ADCOX); - tcg_gen_movi_i32(cpu_tmp3_i32, CC_OP_EFLAGS); - tcg_gen_movcond_i32(TCG_COND_NE, cpu_cc_op, t1, t0, - cpu_tmp2_i32, cpu_tmp3_i32); - tcg_temp_free_i32(t0); - tcg_temp_free_i32(t1); - -- /* The CC_OP value is no longer predictable. */ -+ /* The CC_OP value is no longer predictable. */ - set_cc_op(s, CC_OP_DYNAMIC); - } - -@@ -1686,7 +1759,7 @@ - gen_op_ld_v(s, ot, cpu_T[0], cpu_A0); - else - gen_op_mov_v_reg(ot, cpu_T[0], op1); -- -+ - if (is_right) { - switch (ot) { - case MO_8: -@@ -2252,13 +2325,13 @@ +@@ -2190,13 +2252,13 @@ } static void gen_cmovcc1(CPUX86State *env, DisasContext *s, TCGMemOp ot, int b, @@ -261,211 +207,86 @@ gen_ldst_modrm(env, s, modrm, ot, OR_TMP0, 0); -- cc = gen_prepare_cc(s, b, cpu_T[1]); -+ cc = gen_prepare_cc(s, b, cpu_T[1], cc_optimize_flag); +- cc = gen_prepare_cc(s, b, cpu_T1); ++ cc = gen_prepare_cc(s, b, cpu_T1, cc_optimize_flag); if (cc.mask != -1) { TCGv t0 = tcg_temp_new(); tcg_gen_andi_tl(t0, cc.reg, cc.mask); -@@ -2282,17 +2355,17 @@ - - static inline void gen_op_movl_T0_seg(int seg_reg) - { -- tcg_gen_ld32u_tl(cpu_T[0], cpu_env, -+ tcg_gen_ld32u_tl(cpu_T[0], cpu_env, - offsetof(CPUX86State,segs[seg_reg].selector)); - } - - static inline void gen_op_movl_seg_T0_vm(int seg_reg) - { - tcg_gen_andi_tl(cpu_T[0], cpu_T[0], 0xffff); -- tcg_gen_st32_tl(cpu_T[0], cpu_env, -+ tcg_gen_st32_tl(cpu_T[0], cpu_env, - offsetof(CPUX86State,segs[seg_reg].selector)); - tcg_gen_shli_tl(cpu_T[0], cpu_T[0], 4); -- tcg_gen_st_tl(cpu_T[0], cpu_env, -+ tcg_gen_st_tl(cpu_T[0], cpu_env, - offsetof(CPUX86State,segs[seg_reg].base)); - } - -@@ -3089,7 +3162,7 @@ - #endif - { - gen_ldst_modrm(env, s, modrm, MO_32, OR_TMP0, 0); -- tcg_gen_addi_ptr(cpu_ptr0, cpu_env, -+ tcg_gen_addi_ptr(cpu_ptr0, cpu_env, - offsetof(CPUX86State,fpregs[reg].mmx)); - tcg_gen_trunc_tl_i32(cpu_tmp2_i32, cpu_T[0]); - gen_helper_movl_mm_T0_mmx(cpu_ptr0, cpu_tmp2_i32); -@@ -3099,14 +3172,14 @@ - #ifdef TARGET_X86_64 - if (s->dflag == MO_64) { - gen_ldst_modrm(env, s, modrm, MO_64, OR_TMP0, 0); -- tcg_gen_addi_ptr(cpu_ptr0, cpu_env, -+ tcg_gen_addi_ptr(cpu_ptr0, cpu_env, - offsetof(CPUX86State,xmm_regs[reg])); - gen_helper_movq_mm_T0_xmm(cpu_ptr0, cpu_T[0]); - } else - #endif - { - gen_ldst_modrm(env, s, modrm, MO_32, OR_TMP0, 0); -- tcg_gen_addi_ptr(cpu_ptr0, cpu_env, -+ tcg_gen_addi_ptr(cpu_ptr0, cpu_env, - offsetof(CPUX86State,xmm_regs[reg])); - tcg_gen_trunc_tl_i32(cpu_tmp2_i32, cpu_T[0]); - gen_helper_movl_mm_T0_xmm(cpu_ptr0, cpu_tmp2_i32); -@@ -3263,13 +3336,13 @@ - case 0x7e: /* movd ea, mm */ - #ifdef TARGET_X86_64 - if (s->dflag == MO_64) { -- tcg_gen_ld_i64(cpu_T[0], cpu_env, -+ tcg_gen_ld_i64(cpu_T[0], cpu_env, - offsetof(CPUX86State,fpregs[reg].mmx)); - gen_ldst_modrm(env, s, modrm, MO_64, OR_TMP0, 1); - } else - #endif - { -- tcg_gen_ld32u_tl(cpu_T[0], cpu_env, -+ tcg_gen_ld32u_tl(cpu_T[0], cpu_env, - offsetof(CPUX86State,fpregs[reg].mmx.MMX_L(0))); - gen_ldst_modrm(env, s, modrm, MO_32, OR_TMP0, 1); - } -@@ -3277,13 +3350,13 @@ - case 0x17e: /* movd ea, xmm */ - #ifdef TARGET_X86_64 - if (s->dflag == MO_64) { -- tcg_gen_ld_i64(cpu_T[0], cpu_env, -+ tcg_gen_ld_i64(cpu_T[0], cpu_env, - offsetof(CPUX86State,xmm_regs[reg].XMM_Q(0))); - gen_ldst_modrm(env, s, modrm, MO_64, OR_TMP0, 1); - } else - #endif - { -- tcg_gen_ld32u_tl(cpu_T[0], cpu_env, -+ tcg_gen_ld32u_tl(cpu_T[0], cpu_env, - offsetof(CPUX86State,xmm_regs[reg].XMM_L(0))); - gen_ldst_modrm(env, s, modrm, MO_32, OR_TMP0, 1); - } -@@ -3408,14 +3481,14 @@ - break; - case 0x050: /* movmskps */ - rm = (modrm & 7) | REX_B(s); -- tcg_gen_addi_ptr(cpu_ptr0, cpu_env, -+ tcg_gen_addi_ptr(cpu_ptr0, cpu_env, - offsetof(CPUX86State,xmm_regs[rm])); - gen_helper_movmskps(cpu_tmp2_i32, cpu_env, cpu_ptr0); - tcg_gen_extu_i32_tl(cpu_regs[reg], cpu_tmp2_i32); - break; - case 0x150: /* movmskpd */ - rm = (modrm & 7) | REX_B(s); -- tcg_gen_addi_ptr(cpu_ptr0, cpu_env, -+ tcg_gen_addi_ptr(cpu_ptr0, cpu_env, - offsetof(CPUX86State,xmm_regs[rm])); - gen_helper_movmskpd(cpu_tmp2_i32, cpu_env, cpu_ptr0); - tcg_gen_extu_i32_tl(cpu_regs[reg], cpu_tmp2_i32); -@@ -4411,6 +4484,7 @@ +@@ -4427,6 +4489,7 @@ int modrm, reg, rm, mod, op, opreg, val; target_ulong next_eip, tval; int rex_w, rex_r; + int jcc_op, cc_optimize_flag; - if (unlikely(qemu_loglevel_mask(CPU_LOG_TB_OP | CPU_LOG_TB_OP_OPT))) { - tcg_gen_debug_insn_start(pc_start); -@@ -5000,15 +5074,33 @@ - - case 0x84: /* test Ev, Gv */ - case 0x85: -+ /* Test instruction with two register operands, thus the target of -+ * Chatkey instrumentation */ - ot = mo_b_d(b, dflag); + s->pc_start = s->pc = pc_start; + prefixes = 0; +@@ -5056,10 +5119,23 @@ modrm = cpu_ldub_code(env, s->pc++); reg = ((modrm >> 3) & 7) | rex_r; -+ /* Added to check equality with 'reg' below */ ++ // Added to check equality with 'reg' below. + rm = (modrm & 7) | REX_B(s); gen_ldst_modrm(env, s, modrm, ot, OR_TMP0, 0); - gen_op_mov_v_reg(ot, cpu_T[1], reg); + gen_op_mov_v_reg(ot, cpu_T1, reg); - gen_op_testl_T0_T1_cc(); -- set_cc_op(s, CC_OP_LOGICB + ot); + /* We only focus on test instruction with two same registers as operand, -+ * e.g. "test eax, eax". If so, call gen_op_testl_T0_T1_cc_chatkey(), -+ * instead of gen_op_testl_T0_T1_cc(). ++ * e.g. "test eax, eax". + */ -+ + if (reg == rm && chatkey_EP_passed) { -+ gen_op_testl_T0_T1_cc_chatkey(ot); -+ // Update the parameter index of the latest cmp instruction. -+ assert(tcg_ctx.gen_next_parm_idx >= 2); -+ s->latest_tgt_parm_idx = tcg_ctx.gen_next_parm_idx - 2; -+ set_cc_op(s, CC_OP_LOGICB + ot); ++ gen_op_testl_T0_T1_cc_chatkey(ot); ++ // Record the index of comparison type argument. ++ assert(tcg_ctx.gen_next_parm_idx >= 2); ++ s->latest_tgt_parm_idx = tcg_ctx.gen_next_parm_idx - 2; + } + else { + gen_op_testl_T0_T1_cc(); -+ set_cc_op(s, CC_OP_LOGICB + ot); + } + set_cc_op(s, CC_OP_LOGICB + ot); break; - case 0xa8: /* test eAX, Iv */ -@@ -5217,7 +5309,7 @@ - gen_lea_modrm(env, s, modrm); - gen_helper_cmpxchg16b(cpu_env, cpu_A0); - } else --#endif -+#endif - { - if (!(s->cpuid_features & CPUID_CX8)) - goto illegal_op; -@@ -6276,7 +6368,7 @@ - case 0x6d: - ot = mo_b_d32(b, dflag); - tcg_gen_ext16u_tl(cpu_T[0], cpu_regs[R_EDX]); -- gen_check_io(s, ot, pc_start - s->cs_base, -+ gen_check_io(s, ot, pc_start - s->cs_base, - SVM_IOIO_TYPE_MASK | svm_is_rep(prefixes) | 4); - if (prefixes & (PREFIX_REPZ | PREFIX_REPNZ)) { - gen_repz_ins(s, ot, pc_start - s->cs_base, s->pc - s->cs_base); -@@ -6537,18 +6629,50 @@ +@@ -6569,18 +6645,50 @@ break; case 0x190 ... 0x19f: /* setcc Gv */ + jcc_op = (b >> 1) & 7; + cc_optimize_flag = 1; -+ -+ /* 'cmp' in front of 'setz' is Chatkey's instrumentation target */ ++ /* If this is a SETCC instruction followed by sub/cmp/test, replace the ++ * previous comparison type 'CHATKEY_IGNORE' with a new one, to indicate ++ * that this is our instrumentation target. ++ */ + if (jcc_op == JCC_Z && s->latest_tgt_parm_idx != -1) { + tcg_ctx.gen_opparam_buf[s->latest_tgt_parm_idx] = CHATKEY_CMP_EQUALITY; -+ cc_optimize_flag = 0; // Disable cc optimization ++ cc_optimize_flag = 0; + } else if ((jcc_op == JCC_B || jcc_op == JCC_BE) && s->latest_tgt_parm_idx != -1) { + tcg_ctx.gen_opparam_buf[s->latest_tgt_parm_idx] = CHATKEY_CMP_SIZE_UNSIGNED; -+ cc_optimize_flag = 0; // Disable cc optimization ++ cc_optimize_flag = 0; + } else if ((jcc_op == JCC_L || jcc_op == JCC_LE) && s->latest_tgt_parm_idx != -1) { + tcg_ctx.gen_opparam_buf[s->latest_tgt_parm_idx] = CHATKEY_CMP_SIZE_SIGNED; -+ cc_optimize_flag = 0; // Disable cc optimization ++ cc_optimize_flag = 0; + } -+ modrm = cpu_ldub_code(env, s->pc++); -- gen_setcc1(s, b, cpu_T[0]); -+ gen_setcc1(s, b, cpu_T[0], cc_optimize_flag); +- gen_setcc1(s, b, cpu_T0); ++ gen_setcc1(s, b, cpu_T0, cc_optimize_flag); gen_ldst_modrm(env, s, modrm, MO_8, OR_TMP0, 1); -+ break; case 0x140 ... 0x14f: /* cmov Gv, Ev */ + jcc_op = (b >> 1) & 7; + cc_optimize_flag = 1; -+ -+ /* 'cmp' in front of 'cmovz' is Chatkey's instrumentation target */ ++ /* If this is a CMOV instruction followed by sub/cmp/test, replace the ++ * previous comparison type 'CHATKEY_IGNORE' with a new one, to indicate ++ * that this is our instrumentation target. ++ */ + if (jcc_op == JCC_Z && s->latest_tgt_parm_idx != -1) { + tcg_ctx.gen_opparam_buf[s->latest_tgt_parm_idx] = CHATKEY_CMP_EQUALITY; -+ cc_optimize_flag = 0; // Disable cc optimization ++ cc_optimize_flag = 0; + } else if ((jcc_op == JCC_B || jcc_op == JCC_BE) && s->latest_tgt_parm_idx != -1) { + tcg_ctx.gen_opparam_buf[s->latest_tgt_parm_idx] = CHATKEY_CMP_SIZE_UNSIGNED; -+ cc_optimize_flag = 0; // Disable cc optimization ++ cc_optimize_flag = 0; + } else if ((jcc_op == JCC_L || jcc_op == JCC_LE) && s->latest_tgt_parm_idx != -1) { + tcg_ctx.gen_opparam_buf[s->latest_tgt_parm_idx] = CHATKEY_CMP_SIZE_SIGNED; -+ cc_optimize_flag = 0; // Disable cc optimization ++ cc_optimize_flag = 0; + } -+ -+ if (!(s->cpuid_features & CPUID_CMOV)) { goto illegal_op; } @@ -477,35 +298,13 @@ break; /************************/ -@@ -7338,7 +7462,7 @@ - break; - case 4: /* STGI */ - if ((!(s->flags & HF_SVME_MASK) && -- !(s->cpuid_ext3_features & CPUID_EXT3_SKINIT)) || -+ !(s->cpuid_ext3_features & CPUID_EXT3_SKINIT)) || - !s->pe) - goto illegal_op; - if (s->cpl != 0) { -@@ -7359,8 +7483,8 @@ - } - break; - case 6: /* SKINIT */ -- if ((!(s->flags & HF_SVME_MASK) && -- !(s->cpuid_ext3_features & CPUID_EXT3_SKINIT)) || -+ if ((!(s->flags & HF_SVME_MASK) && -+ !(s->cpuid_ext3_features & CPUID_EXT3_SKINIT)) || - !s->pe) - goto illegal_op; - gen_helper_skinit(cpu_env); -@@ -7978,6 +8102,11 @@ +@@ -8445,6 +8553,9 @@ printf("ERROR addseg\n"); #endif -+ /* (For Chatkey) -+ * Initialize with -1, indicating no 'cmp' or 'test' was encountered yet. -+ */ ++ // Initialize with -1, indicating no 'sub', 'cmp' or 'test' was met yet. + dc->latest_tgt_parm_idx = -1; + - cpu_T[0] = tcg_temp_new(); - cpu_T[1] = tcg_temp_new(); + cpu_T0 = tcg_temp_new(); + cpu_T1 = tcg_temp_new(); cpu_A0 = tcg_temp_new(); diff --git a/Instrumentor/qemu/patches-common/configure.diff b/Instrumentor/qemu/patches-common/configure.diff index 88d3e28..00b1db1 100644 --- a/Instrumentor/qemu/patches-common/configure.diff +++ b/Instrumentor/qemu/patches-common/configure.diff @@ -1,34 +1,11 @@ ---- qemu-2.3.0/configure.orig 2015-04-27 23:08:23.000000000 +0900 -+++ qemu-2.3.0/configure 2019-04-24 16:08:53.331555843 +0900 -@@ -4159,6 +4159,20 @@ - fi +--- qemu-2.10.0/configure.orig 2020-10-01 07:50:32.380129897 -0700 ++++ qemu-2.10.0/configure 2020-10-02 05:01:04.948387878 -0700 +@@ -3855,7 +3855,7 @@ + # check if memfd is supported + memfd=no + cat > $TMPC << EOF +-#include ++#include - ########################################## -+# check for sysmacros.h -+ -+have_sysmacros=no -+cat > $TMPC << EOF -+#include -+int main(void) { -+ return makedev(0, 0); -+} -+EOF -+if compile_prog "" "" ; then -+ have_sysmacros=yes -+fi -+ -+########################################## - # End of CC checks - # After here, no more $cc or $ld runs - -@@ -4956,6 +4970,10 @@ - echo "CONFIG_RDMA=y" >> $config_host_mak - fi - -+if test "$have_sysmacros" = "yes" ; then -+ echo "CONFIG_SYSMACROS=y" >> $config_host_mak -+fi -+ - # Hold two types of flag: - # CONFIG_THREAD_SETNAME_BYTHREAD - we've got a way of setting the name on - # a thread we have a handle to + int main(void) + { diff --git a/Instrumentor/qemu/patches-common/elfload.diff b/Instrumentor/qemu/patches-common/elfload.diff index 2f53f5a..c88ca7c 100644 --- a/Instrumentor/qemu/patches-common/elfload.diff +++ b/Instrumentor/qemu/patches-common/elfload.diff @@ -1,6 +1,6 @@ ---- qemu-2.3.0/linux-user/elfload.c.orig 2017-04-20 20:24:02.721934292 +0900 -+++ qemu-2.3.0/linux-user/elfload.c 2017-04-20 21:49:35.202993175 +0900 -@@ -28,6 +28,8 @@ +--- qemu-2.10.0/linux-user/elfload.c.orig 2020-10-01 07:50:32.384129945 -0700 ++++ qemu-2.10.0/linux-user/elfload.c 2020-10-02 05:01:04.956387921 -0700 +@@ -20,6 +20,8 @@ #define ELF_OSABI ELFOSABI_SYSV @@ -9,40 +9,7 @@ /* from personality.h */ /* -@@ -1339,7 +1341,7 @@ - abi_ulong p) - { - char *tmp, *tmp1, *pag = NULL; -- int len, offset = 0; -+ int padding, len, offset = 0; - - if (!p) { - return 0; /* bullet-proofing */ -@@ -1352,10 +1354,22 @@ - } - tmp1 = tmp; - while (*tmp++); -- len = tmp - tmp1; -+ len = tmp - tmp1; /* Here, len includes '\0' character */ -+ - if (p < len) { /* this shouldn't happen - 128kB */ - return 0; - } -+ -+ /* For Chatkey's node coverage instrumentation. Function strcmp_sse() is -+ * sensitive to the alignment of input string, and it will execute -+ * different code block when input string alignment gets different. To -+ * avoid such unwanted side effect on node coverage, we force the -+ * argv[i] and env[i] strings to be aligned in multiple of 0x40. -+ */ -+ padding = ((p-len) & 0x3f); -+ p -= padding; -+ offset -= padding; -+ - while (len) { - --p; --tmp; --len; - if (--offset < 0) { -@@ -1889,6 +1903,8 @@ +@@ -2085,6 +2087,8 @@ info->brk = 0; info->elf_flags = ehdr->e_flags; diff --git a/Instrumentor/qemu/patches-common/linuxload.diff b/Instrumentor/qemu/patches-common/linuxload.diff deleted file mode 100644 index bc504ee..0000000 --- a/Instrumentor/qemu/patches-common/linuxload.diff +++ /dev/null @@ -1,22 +0,0 @@ ---- qemu-2.3.0/linux-user/linuxload.c.orig 2017-08-14 23:47:29.085927995 +0900 -+++ qemu-2.3.0/linux-user/linuxload.c 2017-08-15 20:31:35.170619303 +0900 -@@ -114,6 +114,9 @@ - put_user_ual(stringp, argv); - argv += n; - stringp += target_strlen(stringp) + 1; -+ if ((stringp & 0x3f) != 0) { -+ stringp += 0x40 - (stringp & 0x3f); -+ } - } - ts->info->arg_end = stringp; - /* FIXME - handle put_user() failures */ -@@ -123,6 +126,9 @@ - put_user_ual(stringp, envp); - envp += n; - stringp += target_strlen(stringp) + 1; -+ if ((stringp & 0x3f) != 0) { -+ stringp += 0x40 - (stringp & 0x3f); -+ } - } - /* FIXME - handle put_user() failures */ - put_user_ual(0, envp); diff --git a/Instrumentor/qemu/patches-common/memfd.diff b/Instrumentor/qemu/patches-common/memfd.diff new file mode 100644 index 0000000..9cce97c --- /dev/null +++ b/Instrumentor/qemu/patches-common/memfd.diff @@ -0,0 +1,13 @@ +--- qemu-2.10.0/util/memfd.c.orig 2020-10-01 07:50:32.384129945 -0700 ++++ qemu-2.10.0/util/memfd.c 2020-10-02 05:01:04.956387921 -0700 +@@ -31,9 +31,7 @@ + + #include "qemu/memfd.h" + +-#ifdef CONFIG_MEMFD +-#include +-#elif defined CONFIG_LINUX ++#if defined CONFIG_LINUX && !defined CONFIG_MEMFD + #include + #include + diff --git a/Instrumentor/qemu/patches-common/os-posix.diff b/Instrumentor/qemu/patches-common/os-posix.diff deleted file mode 100644 index 460eedb..0000000 --- a/Instrumentor/qemu/patches-common/os-posix.diff +++ /dev/null @@ -1,13 +0,0 @@ ---- qemu-2.3.0/include/sysemu/os-posix.h.orig 2015-04-27 23:08:25.000000000 +0900 -+++ qemu-2.3.0/include/sysemu/os-posix.h 2019-04-24 16:08:53.331555843 +0900 -@@ -28,6 +28,10 @@ - - #include - -+#ifdef CONFIG_SYSMACROS -+#include -+#endif -+ - void os_set_line_buffering(void); - void os_set_proc_name(const char *s); - void os_setup_signal_handling(void); diff --git a/Instrumentor/qemu/patches-common/signal.diff b/Instrumentor/qemu/patches-common/signal.diff index efb691e..950918c 100644 --- a/Instrumentor/qemu/patches-common/signal.diff +++ b/Instrumentor/qemu/patches-common/signal.diff @@ -1,187 +1,26 @@ ---- qemu-2.3.0/linux-user/signal.c.orig 2018-09-14 14:35:01.547517238 +0900 -+++ qemu-2.3.0/linux-user/signal.c 2018-09-14 14:53:42.013375909 +0900 -@@ -30,6 +30,8 @@ - #include "qemu-common.h" +--- qemu-2.10.0/linux-user/signal.c.orig 2020-10-01 07:50:32.384129945 -0700 ++++ qemu-2.10.0/linux-user/signal.c 2020-10-02 05:04:07.337597365 -0700 +@@ -26,6 +26,8 @@ #include "target_signal.h" + #include "trace.h" +extern void chatkey_exit(void); + - //#define DEBUG_SIGNAL - static struct target_sigaltstack target_sigaltstack_used = { -@@ -500,6 +502,16 @@ - abi_ulong handler; - int queue; - -+ /* If the signal indicates (1) what we define as 'crash'(e.g. -+ * segfault, abortion) (2) or termination due to timeout, call -+ * chatkey_exit() to flush out the information traced until now. + .ss_sp = 0, + .ss_size = 0, +@@ -6563,6 +6565,14 @@ + struct target_sigaction *sa; + TaskState *ts = cpu->opaque; + ++ /* If the signal indicates a crash or timeout, call chatkey_exit() to flush ++ * out the information traced until now. + */ + if (sig == SIGSEGV || sig == SIGFPE || sig == SIGILL || sig == SIGABRT || + sig == SIGTERM) { + chatkey_exit(); + } + -+ - #if defined(DEBUG_SIGNAL) - fprintf(stderr, "queue_signal: sig=%d\n", - sig); -@@ -1062,7 +1074,7 @@ - - fpstate_addr = tswapl(sc->fpstate); - if (fpstate_addr != 0) { -- if (!access_ok(VERIFY_READ, fpstate_addr, -+ if (!access_ok(VERIFY_READ, fpstate_addr, - sizeof(struct target_fpstate))) - goto badframe; - cpu_x86_frstor(env, fpstate_addr, 1); -@@ -1124,7 +1136,7 @@ - if (restore_sigcontext(env, &frame->uc.tuc_mcontext, &eax)) - goto badframe; - -- if (do_sigaltstack(frame_addr + offsetof(struct rt_sigframe, uc.tuc_stack), 0, -+ if (do_sigaltstack(frame_addr + offsetof(struct rt_sigframe, uc.tuc_stack), 0, - get_sp_from_cpustate(env)) == -EFAULT) - goto badframe; - -@@ -2216,7 +2228,7 @@ - #define UREG_FP UREG_I6 - #define UREG_SP UREG_O6 - --static inline abi_ulong get_sigframe(struct target_sigaction *sa, -+static inline abi_ulong get_sigframe(struct target_sigaction *sa, - CPUSPARCState *env, - unsigned long framesize) - { -@@ -2285,11 +2297,11 @@ - sigframe_size = NF_ALIGNEDSZ; - sf_addr = get_sigframe(ka, env, sigframe_size); - -- sf = lock_user(VERIFY_WRITE, sf_addr, -+ sf = lock_user(VERIFY_WRITE, sf_addr, - sizeof(struct target_signal_frame), 0); - if (!sf) - goto sigsegv; -- -+ - //fprintf(stderr, "sf: %x pc %x fp %x sp %x\n", sf, env->pc, env->regwptr[UREG_FP], env->regwptr[UREG_SP]); - #if 0 - if (invalid_frame_pointer(sf, sigframe_size)) -@@ -2319,9 +2331,9 @@ - /* 3. signal handler back-trampoline and parameters */ - env->regwptr[UREG_FP] = sf_addr; - env->regwptr[UREG_I0] = sig; -- env->regwptr[UREG_I1] = sf_addr + -+ env->regwptr[UREG_I1] = sf_addr + - offsetof(struct target_signal_frame, info); -- env->regwptr[UREG_I2] = sf_addr + -+ env->regwptr[UREG_I2] = sf_addr + - offsetof(struct target_signal_frame, info); - - /* 4. signal handler */ -@@ -2333,7 +2345,7 @@ - else { - uint32_t val32; - -- env->regwptr[UREG_I7] = sf_addr + -+ env->regwptr[UREG_I7] = sf_addr + - offsetof(struct target_signal_frame, insns) - 2 * 4; - - /* mov __NR_sigreturn, %g1 */ -@@ -2578,10 +2590,10 @@ - __get_user(i7, &(ucp->tuc_mcontext.mc_i7)); - - w_addr = TARGET_STACK_BIAS+env->regwptr[UREG_I6]; -- if (put_user(fp, w_addr + offsetof(struct target_reg_window, ins[6]), -+ if (put_user(fp, w_addr + offsetof(struct target_reg_window, ins[6]), - abi_ulong) != 0) - goto do_sigsegv; -- if (put_user(i7, w_addr + offsetof(struct target_reg_window, ins[7]), -+ if (put_user(i7, w_addr + offsetof(struct target_reg_window, ins[7]), - abi_ulong) != 0) - goto do_sigsegv; - /* FIXME this does not match how the kernel handles the FPU in -@@ -2626,7 +2638,7 @@ - ucp_addr = env->regwptr[UREG_I0]; - if (!lock_user_struct(VERIFY_WRITE, ucp, ucp_addr, 0)) - goto do_sigsegv; -- -+ - mcp = &ucp->tuc_mcontext; - grp = &mcp->mc_gregs; - -@@ -2675,10 +2687,10 @@ - - w_addr = TARGET_STACK_BIAS+env->regwptr[UREG_I6]; - fp = i7 = 0; -- if (get_user(fp, w_addr + offsetof(struct target_reg_window, ins[6]), -+ if (get_user(fp, w_addr + offsetof(struct target_reg_window, ins[6]), - abi_ulong) != 0) - goto do_sigsegv; -- if (get_user(i7, w_addr + offsetof(struct target_reg_window, ins[7]), -+ if (get_user(i7, w_addr + offsetof(struct target_reg_window, ins[7]), - abi_ulong) != 0) - goto do_sigsegv; - __put_user(fp, &(mcp->mc_fp)); -@@ -3417,7 +3429,7 @@ - - struct rt_signal_frame { - siginfo_t info; -- struct ucontext uc; -+ ucontext_t uc; - uint32_t tramp[2]; - }; - -@@ -3596,7 +3608,7 @@ - /* We got here through a sigreturn syscall, our path back is via an - rtb insn so setup r14 for that. */ - env->regs[14] = env->sregs[SR_PC]; -- -+ - unlock_user_struct(frame, frame_addr, 0); - return env->regs[10]; - badframe: -@@ -3628,7 +3640,7 @@ - siginfo_t *pinfo; - void *puc; - siginfo_t info; -- struct ucontext uc; -+ ucontext_t uc; - uint16_t retcode[4]; /* Trampoline code. */ - }; - -@@ -3931,7 +3943,7 @@ - tswap_siginfo(&frame->info, info); - } - -- /*err |= __clear_user(&frame->uc, offsetof(struct ucontext, uc_mcontext));*/ -+ /*err |= __clear_user(&frame->uc, offsetof(ucontext_t, uc_mcontext));*/ - __put_user(0, &frame->uc.tuc_flags); - __put_user(0, &frame->uc.tuc_link); - __put_user(target_sigaltstack_used.ss_sp, -@@ -4948,7 +4960,7 @@ - abi_ulong extramask[TARGET_NSIG_WORDS-1]; - struct target_sigcontext sc; - }; -- -+ - typedef int target_greg_t; - #define TARGET_NGREG 18 - typedef target_greg_t target_gregset_t[TARGET_NGREG]; -@@ -5107,14 +5119,14 @@ - - return 0; - } -- -+ - static inline int target_rt_restore_ucontext(CPUM68KState *env, - struct target_ucontext *uc, - int *pd0) - { - int temp; - target_greg_t *gregs = uc->tuc_mcontext.gregs; -- -+ - __get_user(temp, &uc->tuc_mcontext.version); - if (temp != TARGET_MCONTEXT_VERSION) - goto badframe; + trace_user_handle_signal(cpu_env, sig); + /* dequeue signal */ + k->pending = 0; diff --git a/Instrumentor/qemu/patches-common/texi2pod.diff b/Instrumentor/qemu/patches-common/texi2pod.diff deleted file mode 100644 index 6e3dfdf..0000000 --- a/Instrumentor/qemu/patches-common/texi2pod.diff +++ /dev/null @@ -1,11 +0,0 @@ ---- qemu-2.3.0/scripts/texi2pod.pl.orig 2018-09-14 14:21:57.762531279 +0900 -+++ qemu-2.3.0/scripts/texi2pod.pl 2018-09-14 14:22:08.914429351 +0900 -@@ -317,7 +317,7 @@ - @columns = (); - for $column (split (/\s*\@tab\s*/, $1)) { - # @strong{...} is used a @headitem work-alike -- $column =~ s/^\@strong{(.*)}$/$1/; -+ $column =~ s/^\@strong\{(.*)\}$/$1/; - push @columns, $column; - } - $_ = "\n=item ".join (" : ", @columns)."\n"; diff --git a/Instrumentor/qemu/patches-common/translate-all.diff b/Instrumentor/qemu/patches-common/translate-all.diff deleted file mode 100644 index 180b3a5..0000000 --- a/Instrumentor/qemu/patches-common/translate-all.diff +++ /dev/null @@ -1,18 +0,0 @@ ---- qemu-2.3.0/translate-all.c.orig 2014-12-09 14:45:46.000000000 +0000 -+++ qemu-2.3.0/translate-all.c 2015-01-28 22:37:42.383000000 +0000 -@@ -393,8 +393,13 @@ - /* We can't use g_malloc because it may recurse into a locked mutex. */ - # define ALLOC(P, SIZE) \ - do { \ -- P = mmap(NULL, SIZE, PROT_READ | PROT_WRITE, \ -- MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); \ -+ void* _tmp = mmap(NULL, SIZE, PROT_READ | PROT_WRITE, \ -+ MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); \ -+ if (_tmp == (void*)-1) { \ -+ qemu_log(">>> Out of memory for stack, bailing out. <<<\n"); \ -+ exit(1); \ -+ } \ -+ (P) = _tmp; \ - } while (0) - #else - # define ALLOC(P, SIZE) \ diff --git a/Instrumentor/qemu/patches-common/user-exec.diff b/Instrumentor/qemu/patches-common/user-exec.diff deleted file mode 100644 index 851d65d..0000000 --- a/Instrumentor/qemu/patches-common/user-exec.diff +++ /dev/null @@ -1,119 +0,0 @@ ---- qemu-2.3.0/user-exec.c.orig 2018-09-14 14:44:20.264077037 +0900 -+++ qemu-2.3.0/user-exec.c 2018-09-14 14:45:52.166948137 +0900 -@@ -57,7 +57,7 @@ - void cpu_resume_from_signal(CPUState *cpu, void *puc) - { - #ifdef __linux__ -- struct ucontext *uc = puc; -+ ucontext_t *uc = puc; - #elif defined(__OpenBSD__) - struct sigcontext *uc = puc; - #endif -@@ -171,7 +171,7 @@ - #elif defined(__OpenBSD__) - struct sigcontext *uc = puc; - #else -- struct ucontext *uc = puc; -+ ucontext_t *uc = puc; - #endif - unsigned long pc; - int trapno; -@@ -226,7 +226,7 @@ - #elif defined(__OpenBSD__) - struct sigcontext *uc = puc; - #else -- struct ucontext *uc = puc; -+ ucontext_t *uc = puc; - #endif - - pc = PC_sig(uc); -@@ -288,7 +288,7 @@ - - #ifdef __APPLE__ - #include --typedef struct ucontext SIGCONTEXT; -+typedef ucontext_t SIGCONTEXT; - /* All Registers access - only for local access */ - #define REG_sig(reg_name, context) \ - ((context)->uc_mcontext->ss.reg_name) -@@ -331,7 +331,7 @@ - #if defined(__FreeBSD__) || defined(__FreeBSD_kernel__) - ucontext_t *uc = puc; - #else -- struct ucontext *uc = puc; -+ ucontext_t *uc = puc; - #endif - unsigned long pc; - int is_write; -@@ -358,7 +358,7 @@ - void *puc) - { - siginfo_t *info = pinfo; -- struct ucontext *uc = puc; -+ ucontext_t *uc = puc; - uint32_t *pc = uc->uc_mcontext.sc_pc; - uint32_t insn = *pc; - int is_write = 0; -@@ -456,7 +456,7 @@ - #if defined(__NetBSD__) - ucontext_t *uc = puc; - #else -- struct ucontext *uc = puc; -+ ucontext_t *uc = puc; - #endif - unsigned long pc; - int is_write; -@@ -483,7 +483,7 @@ - int cpu_signal_handler(int host_signum, void *pinfo, void *puc) - { - siginfo_t *info = pinfo; -- struct ucontext *uc = puc; -+ ucontext_t *uc = puc; - uintptr_t pc = uc->uc_mcontext.pc; - uint32_t insn = *(uint32_t *)pc; - bool is_write; -@@ -512,7 +512,7 @@ - void *puc) - { - siginfo_t *info = pinfo; -- struct ucontext *uc = puc; -+ ucontext_t *uc = puc; - unsigned long pc; - int is_write; - -@@ -534,7 +534,7 @@ - int cpu_signal_handler(int host_signum, void *pinfo, void *puc) - { - siginfo_t *info = pinfo; -- struct ucontext *uc = puc; -+ ucontext_t *uc = puc; - unsigned long ip; - int is_write = 0; - -@@ -565,7 +565,7 @@ - void *puc) - { - siginfo_t *info = pinfo; -- struct ucontext *uc = puc; -+ ucontext_t *uc = puc; - unsigned long pc; - uint16_t *pinsn; - int is_write = 0; -@@ -618,7 +618,7 @@ - void *puc) - { - siginfo_t *info = pinfo; -- struct ucontext *uc = puc; -+ ucontext_t *uc = puc; - greg_t pc = uc->uc_mcontext.pc; - int is_write; - -@@ -634,7 +634,7 @@ - void *puc) - { - siginfo_t *info = pinfo; -- struct ucontext *uc = puc; -+ ucontext_t *uc = puc; - unsigned long pc = uc->uc_mcontext.sc_iaoq[0]; - uint32_t insn = *(uint32_t *)pc; - int is_write = 0; diff --git a/Instrumentor/qemu/patches-coverage/afl-qemu-cpu-inl.h b/Instrumentor/qemu/patches-coverage/afl-qemu-cpu-inl.h index 6185c7a..758b455 100644 --- a/Instrumentor/qemu/patches-coverage/afl-qemu-cpu-inl.h +++ b/Instrumentor/qemu/patches-coverage/afl-qemu-cpu-inl.h @@ -30,14 +30,11 @@ unsigned int afl_forksrv_pid; /* Function declarations. */ -static void afl_forkserver(CPUArchState*); +static void afl_forkserver(CPUState*); -static void afl_wait_tsl(CPUArchState*, int); +static void afl_wait_tsl(CPUState*, int); static void afl_request_tsl(target_ulong, target_ulong, uint64_t); -static TranslationBlock *tb_find_slow(CPUArchState*, target_ulong, - target_ulong, uint64_t); - /* Data structure passed around by the translate handlers: */ struct afl_tsl { @@ -46,14 +43,13 @@ struct afl_tsl { uint64_t flags; }; - /************************* * ACTUAL IMPLEMENTATION * *************************/ /* Fork server logic, invoked once we hit _start. */ -static void afl_forkserver(CPUArchState *env) { +static void afl_forkserver(CPUState *cpu) { static unsigned char tmp[4]; @@ -75,7 +71,7 @@ static void afl_forkserver(CPUArchState *env) { /* Whoops, parent dead? */ - if (read(FORKSRV_FD, &tmp, 4) != 4) exit(2); + if (read(FORKSRV_FD, tmp, 4) != 4) exit(2); /* Establish a channel with child to grab translation commands. We'll read from t_fd[0], child will write to TSL_FD. */ @@ -106,7 +102,7 @@ static void afl_forkserver(CPUArchState *env) { /* Collect translation requests until child dies and closes the pipe. */ - afl_wait_tsl(env, t_fd[0]); + afl_wait_tsl(cpu, t_fd[0]); /* Get and relay exit status to parent. */ @@ -137,13 +133,13 @@ static void afl_request_tsl(target_ulong pc, target_ulong cb, uint64_t flags) { } - /* This is the other side of the same channel. Since timeouts are handled by afl-fuzz simply killing the child, we can just wait until the pipe breaks. */ -static void afl_wait_tsl(CPUArchState *env, int fd) { +static void afl_wait_tsl(CPUState *cpu, int fd) { struct afl_tsl t; + TranslationBlock *tb; while (1) { @@ -152,7 +148,15 @@ static void afl_wait_tsl(CPUArchState *env, int fd) { if (read(fd, &t, sizeof(struct afl_tsl)) != sizeof(struct afl_tsl)) break; - tb_find_slow(env, t.pc, t.cs_base, t.flags); + tb = tb_htable_lookup(cpu, t.pc, t.cs_base, t.flags); + + if(!tb) { + mmap_lock(); + tb_lock(); + tb_gen_code(cpu, t.pc, t.cs_base, t.flags, 0); + mmap_unlock(); + tb_unlock(); + } } diff --git a/Instrumentor/qemu/patches-coverage/chatkey.cc b/Instrumentor/qemu/patches-coverage/chatkey.c similarity index 78% rename from Instrumentor/qemu/patches-coverage/chatkey.cc rename to Instrumentor/qemu/patches-coverage/chatkey.c index db992f4..b920f4b 100644 --- a/Instrumentor/qemu/patches-coverage/chatkey.cc +++ b/Instrumentor/qemu/patches-coverage/chatkey.c @@ -7,6 +7,8 @@ #include #include +#include "qemu/osdep.h" + #ifdef TARGET_X86_64 typedef uint64_t abi_ulong; #else @@ -17,18 +19,25 @@ extern unsigned int afl_forksrv_pid; #define FORKSRV_FD 198 #define TSL_FD (FORKSRV_FD - 1) +void chatkey_setup_before_forkserver(void); +void chatkey_setup_after_forkserver(void); +void chatkey_close_fp(void); +void chatkey_exit(void); +void chatkey_log_bb(abi_ulong addr); + abi_ulong chatkey_entry_point; /* ELF entry point (_start) */ -static int found_new_edge = 0; -static int found_new_path = 0; // TODO. Extend to measure path coverage, too. -static abi_ulong prev_node = 0; static char * coverage_path = NULL; static char * dbg_path = NULL; static FILE * coverage_fp = NULL; static FILE * dbg_fp = NULL; + +static abi_ulong prev_addr = 0; +static int found_new_edge = 0; +static int found_new_path = 0; // TODO. Extend to measure path coverage, too. static unsigned char * edge_bitmap = NULL; -extern "C" void chatkey_setup_before_forkserver(void) { +void chatkey_setup_before_forkserver(void) { char * bitmap_path = getenv("CK_BITMAP_LOG"); int bitmap_fd = open(bitmap_path, O_RDWR | O_CREAT, 0644); edge_bitmap = (unsigned char*) mmap(NULL, 0x10000, PROT_READ | PROT_WRITE, MAP_SHARED, bitmap_fd, 0); @@ -38,7 +47,7 @@ extern "C" void chatkey_setup_before_forkserver(void) { dbg_path = getenv("CK_DBG_LOG"); } -extern "C" void chatkey_setup_after_forkserver(void) { +void chatkey_setup_after_forkserver(void) { /* Open file pointers and descriptors early, since if we try to open them in * chatkey_exit(), it gets mixed with stderr & stdout stream. This seems to * be an issue due to incorrect file descriptor management in QEMU code. @@ -55,7 +64,7 @@ extern "C" void chatkey_setup_after_forkserver(void) { } // When fork() syscall is encountered, child process should call this function. -extern "C" void chatkey_close_fp(void) { +void chatkey_close_fp(void) { // Close file pointers, to avoid dumping log twice. if (coverage_fp) { fclose(coverage_fp); @@ -67,11 +76,16 @@ extern "C" void chatkey_close_fp(void) { dbg_fp = NULL; } + if (edge_bitmap) { + munmap(edge_bitmap, 0x10000); + edge_bitmap = NULL; + } + if (afl_forksrv_pid) close(TSL_FD); } -extern "C" void chatkey_exit(void) { +void chatkey_exit(void) { sigset_t mask; // Block signals, since we register signal handler that calls chatkey_exit()/ @@ -97,20 +111,24 @@ extern "C" void chatkey_exit(void) { } } -extern "C" void chatkey_log_bb(abi_ulong addr) { +void chatkey_log_bb(abi_ulong addr) { + abi_ulong prev_addr_local; abi_ulong edge, hash; unsigned int byte_idx, byte_mask; unsigned char old_byte, new_byte; + // Make sure that 'prev_addr' is always updated even if we just return. + prev_addr_local = prev_addr; + prev_addr = addr; + if (!coverage_fp || !edge_bitmap) return; #ifdef TARGET_X86_64 - edge = (prev_node << 16) ^ addr; + edge = (prev_addr_local << 16) ^ addr; #else - edge = (prev_node << 8) ^ addr; + edge = (prev_addr_local << 8) ^ addr; #endif - prev_node = addr; // Update bitmap. hash = (edge >> 4) ^ (edge << 8); @@ -121,7 +139,7 @@ extern "C" void chatkey_log_bb(abi_ulong addr) { if (old_byte != new_byte) { found_new_edge = 1; edge_bitmap[byte_idx] = new_byte; - /* Log visited addrs if dbg_fp is not NULL */ + /* Log visited nodes if dbg_fp is not NULL */ if (dbg_fp) { #ifdef TARGET_X86_64 fprintf(dbg_fp, "(0x%lx)\n", addr); diff --git a/Instrumentor/qemu/patches-coverage/cpu-exec.diff b/Instrumentor/qemu/patches-coverage/cpu-exec.diff index bea08c3..cc75516 100644 --- a/Instrumentor/qemu/patches-coverage/cpu-exec.diff +++ b/Instrumentor/qemu/patches-coverage/cpu-exec.diff @@ -1,44 +1,41 @@ ---- qemu-2.3.0-coverage/cpu-exec.c.orig 2020-09-21 23:23:17.517246984 -0700 -+++ qemu-2.3.0-coverage/cpu-exec.c 2020-09-21 23:22:27.909433624 -0700 -@@ -28,6 +28,13 @@ - #include "exec/memory-internal.h" - #include "qemu/rcu.h" +--- qemu-2.10.0-coverage/accel/tcg/cpu-exec.c.orig 2020-10-03 21:05:29.913244803 -0700 ++++ qemu-2.10.0-coverage/accel/tcg/cpu-exec.c 2020-10-03 21:05:21.857199845 -0700 +@@ -36,6 +36,12 @@ + #include "sysemu/cpus.h" + #include "sysemu/replay.h" ++#include "afl-qemu-cpu-inl.h" +extern abi_ulong chatkey_entry_point; /* ELF entry point (_start) */ +extern void chatkey_setup_before_forkserver(void); +extern void chatkey_setup_after_forkserver(void); +extern void chatkey_log_bb(abi_ulong addr); -+ -+#include "afl-qemu-cpu-inl.h" + /* -icount align implementation. */ typedef struct SyncClocks { -@@ -296,8 +303,11 @@ - } - not_found: - /* if no translated code available, then translate it now */ +@@ -143,6 +149,17 @@ + TranslationBlock *last_tb; + int tb_exit; + uint8_t *tb_ptr = itb->tc_ptr; ++ abi_ulong entry_pc; + - tb = tb_gen_code(cpu, pc, cs_base, flags, 0); ++ entry_pc = itb->pc; ++ if(entry_pc == chatkey_entry_point) { ++ chatkey_setup_before_forkserver(); ++ // Resolves util/rcu.c assertion error issue (cf. AFL-2.53b). ++ rcu_disable_atfork(); ++ afl_forkserver(cpu); ++ chatkey_setup_after_forkserver(); ++ } ++ chatkey_log_bb(entry_pc); -+ afl_request_tsl(pc, cs_base, flags); -+ - found: - /* Move the last found TB to the head of the list */ - if (likely(*ptb1)) { -@@ -492,6 +502,15 @@ - next_tb = 0; - tcg_ctx.tb_ctx.tb_invalidated_flag = 0; - } -+ -+ if(tb->pc == chatkey_entry_point) { -+ chatkey_setup_before_forkserver(); -+ afl_forkserver(env); -+ chatkey_setup_after_forkserver(); -+ } -+ -+ chatkey_log_bb(tb->pc); -+ - if (qemu_loglevel_mask(CPU_LOG_EXEC)) { - qemu_log("Trace %p [" TARGET_FMT_lx "] %s\n", - tb->tc_ptr, tb->pc, lookup_symbol(tb->pc)); + qemu_log_mask_and_addr(CPU_LOG_EXEC, itb->pc, + "Trace %p [%d: " TARGET_FMT_lx "] %s\n", +@@ -365,6 +382,7 @@ + if (!tb) { + /* if no translated code available, then translate it now */ + tb = tb_gen_code(cpu, pc, cs_base, flags, 0); ++ afl_request_tsl(pc, cs_base, flags); + } + + mmap_unlock(); diff --git a/Instrumentor/qemu/patches-coverage/exec-all.diff b/Instrumentor/qemu/patches-coverage/exec-all.diff deleted file mode 100644 index d96db38..0000000 --- a/Instrumentor/qemu/patches-coverage/exec-all.diff +++ /dev/null @@ -1,12 +0,0 @@ ---- qemu-2.3.0-coverage/include/exec/exec-all.h.orig 2017-10-07 02:08:36.745270225 +0900 -+++ qemu-2.3.0-coverage/include/exec/exec-all.h 2017-10-07 00:34:56.355819708 +0900 -@@ -173,6 +173,9 @@ - jmp_first */ - struct TranslationBlock *jmp_next[2]; - struct TranslationBlock *jmp_first; -+ /* Chatkey : Introduce flags to mark call/return block */ -+ int is_call; -+ int is_ret; - }; - - #include "exec/spinlock.h" diff --git a/Instrumentor/qemu/patches-coverage/makefile-objs.diff b/Instrumentor/qemu/patches-coverage/makefile-objs.diff new file mode 100644 index 0000000..4e6545f --- /dev/null +++ b/Instrumentor/qemu/patches-coverage/makefile-objs.diff @@ -0,0 +1,7 @@ +--- qemu-2.10.0-coverage/accel/tcg/Makefile.objs.orig 2020-10-03 21:05:29.921244848 -0700 ++++ qemu-2.10.0-coverage/accel/tcg/Makefile.objs 2020-10-03 21:05:21.853199823 -0700 +@@ -1,3 +1,3 @@ + obj-$(CONFIG_SOFTMMU) += tcg-all.o + obj-$(CONFIG_SOFTMMU) += cputlb.o +-obj-y += cpu-exec.o cpu-exec-common.o translate-all.o ++obj-y += cpu-exec.o cpu-exec-common.o translate-all.o chatkey.o diff --git a/Instrumentor/qemu/patches-coverage/makefile-target.diff b/Instrumentor/qemu/patches-coverage/makefile-target.diff deleted file mode 100644 index c58722f..0000000 --- a/Instrumentor/qemu/patches-coverage/makefile-target.diff +++ /dev/null @@ -1,20 +0,0 @@ ---- qemu-2.3.0-coverage/Makefile.target 2015-04-27 23:08:23.000000000 +0900 -+++ qemu-2.3.0-coverage-new/Makefile.target 2016-07-12 15:54:44.666065500 +0900 -@@ -11,7 +11,7 @@ - endif - QEMU_CFLAGS += -I.. -I$(SRC_PATH)/target-$(TARGET_BASE_ARCH) -DNEED_CPU_H - --QEMU_CFLAGS+=-I$(SRC_PATH)/include -+QEMU_CFLAGS+=-I$(SRC_PATH)/include -I$(SRC_PATH)/../../sparsehash/sparsehash-2.0.3/src -I$(SRC_PATH)/../../sparsehash/build/src - - ifdef CONFIG_USER_ONLY - # user emulator name -@@ -82,7 +82,7 @@ - - ######################################################### - # cpu emulator library --obj-y = exec.o translate-all.o cpu-exec.o -+obj-y = exec.o translate-all.o cpu-exec.o chatkey.o - obj-y += tcg/tcg.o tcg/tcg-op.o tcg/optimize.o - obj-$(CONFIG_TCG_INTERPRETER) += tci.o - obj-$(CONFIG_TCG_INTERPRETER) += disas/tci.o diff --git a/Instrumentor/qemu/patches-coverage/syscall.diff b/Instrumentor/qemu/patches-coverage/syscall.diff index 8740728..e4f050e 100644 --- a/Instrumentor/qemu/patches-coverage/syscall.diff +++ b/Instrumentor/qemu/patches-coverage/syscall.diff @@ -1,156 +1,25 @@ ---- qemu-2.3.0-coverage/linux-user/syscall.c.orig 2017-07-20 14:40:31.912793680 +0900 -+++ qemu-2.3.0-coverage/linux-user/syscall.c 2017-07-20 14:25:16.001550285 +0900 -@@ -115,6 +115,33 @@ +--- qemu-2.10.0-coverage/linux-user/syscall.c.orig 2020-10-03 21:05:29.921244848 -0700 ++++ qemu-2.10.0-coverage/linux-user/syscall.c 2020-10-03 21:05:21.897200066 -0700 +@@ -116,6 +116,10 @@ #include "qemu.h" +extern void chatkey_exit(void); +extern void chatkey_close_fp(void); -+int check_executable(const char* filename); -+ -+int check_executable(const char* filename) { -+ int fd, size, tmp; -+ if (eaccess(filename, R_OK) == 0 && eaccess(filename, X_OK == 0)) { -+ fd = open(filename, O_RDONLY); -+ if (fd < 0) -+ return -1; -+ size = lseek(fd, 0, SEEK_END); -+ lseek(fd, 0, SEEK_SET); -+ if (size < 4) { -+ close(fd); -+ return -1; -+ } -+ if (read(fd, &tmp, 4) != 4) { -+ close(fd); -+ return -1; -+ } -+ close(fd); -+ if (tmp == 0x464c457f) -+ return 0; -+ } -+ -+ return -1; -+} - #define CLONE_NPTL_FLAGS2 (CLONE_SETTLS | \ - CLONE_PARENT_SETTID | CLONE_CHILD_SETTID | CLONE_CHILD_CLEARTID) - -@@ -227,7 +254,22 @@ - _syscall3(int,sys_rt_sigqueueinfo,int,pid,int,sig,siginfo_t *,uinfo) - _syscall3(int,sys_syslog,int,type,char*,bufp,int,len) - #if defined(TARGET_NR_tgkill) && defined(__NR_tgkill) --_syscall3(int,sys_tgkill,int,tgid,int,pid,int,sig) -+ +extern unsigned int afl_forksrv_pid; + -+static int sys_tgkill(int tgid, int pid, int sig) { -+ -+ /* Workaround for -lpthread to make abort() work properly, without -+ killing the forkserver due to a prematurely cached PID. */ -+ -+ if (afl_forksrv_pid && afl_forksrv_pid == pid && -+ (sig == SIGABRT || sig == SIGSEGV || sig == SIGFPE || sig == SIGILL)) -+ pid = tgid = getpid(); -+ -+ return syscall(__NR_sys_tgkill, pid, tgid, sig); -+ -+} -+ + #ifndef CLONE_IO + #define CLONE_IO 0x80000000 /* Clone io context */ #endif - #if defined(TARGET_NR_tkill) && defined(__NR_tkill) - _syscall2(int,sys_tkill,int,tid,int,sig) -@@ -846,7 +888,7 @@ - { - abi_ulong target_rlim_swap; - rlim_t result; -- -+ - target_rlim_swap = tswapal(target_rlim); - if (target_rlim_swap == TARGET_RLIM_INFINITY) - return RLIM_INFINITY; -@@ -854,7 +896,7 @@ - result = target_rlim_swap; - if (target_rlim_swap != (rlim_t)result) - return RLIM_INFINITY; -- -+ - return result; - } - -@@ -862,13 +904,13 @@ - { - abi_ulong target_rlim_swap; - abi_ulong result; -- -+ - if (rlim == RLIM_INFINITY || rlim != (abi_long)rlim) - target_rlim_swap = TARGET_RLIM_INFINITY; - else - target_rlim_swap = rlim; - result = tswapal(target_rlim_swap); -- -+ - return result; - } - -@@ -1183,9 +1225,9 @@ - abi_ulong target_cmsg_addr; - struct target_cmsghdr *target_cmsg; - socklen_t space = 0; -- -+ - msg_controllen = tswapal(target_msgh->msg_controllen); -- if (msg_controllen < sizeof (struct target_cmsghdr)) -+ if (msg_controllen < sizeof (struct target_cmsghdr)) - goto the_end; - target_cmsg_addr = tswapal(target_msgh->msg_control); - target_cmsg = lock_user(VERIFY_READ, target_cmsg_addr, msg_controllen, 1); -@@ -1255,7 +1297,7 @@ - socklen_t space = 0; - - msg_controllen = tswapal(target_msgh->msg_controllen); -- if (msg_controllen < sizeof (struct target_cmsghdr)) -+ if (msg_controllen < sizeof (struct target_cmsghdr)) - goto the_end; - target_cmsg_addr = tswapal(target_msgh->msg_control); - target_cmsg = lock_user(VERIFY_WRITE, target_cmsg_addr, msg_controllen, 0); -@@ -4293,7 +4335,7 @@ - } - unlock_user_struct(target_ldt_info, ptr, 1); - -- if (ldt_info.entry_number < TARGET_GDT_ENTRY_TLS_MIN || -+ if (ldt_info.entry_number < TARGET_GDT_ENTRY_TLS_MIN || - ldt_info.entry_number > TARGET_GDT_ENTRY_TLS_MAX) - return -TARGET_EINVAL; - seg_32bit = ldt_info.flags & 1; -@@ -4371,7 +4413,7 @@ - lp = (uint32_t *)(gdt_table + idx); - entry_1 = tswap32(lp[0]); - entry_2 = tswap32(lp[1]); -- -+ - read_exec_only = ((entry_2 >> 9) & 1) ^ 1; - contents = (entry_2 >> 10) & 3; - seg_not_present = ((entry_2 >> 15) & 1) ^ 1; -@@ -4387,8 +4429,8 @@ - (read_exec_only << 3) | (limit_in_pages << 4) | - (seg_not_present << 5) | (useable << 6) | (lm << 7); - limit = (entry_1 & 0xffff) | (entry_2 & 0xf0000); -- base_addr = (entry_1 >> 16) | -- (entry_2 & 0xff000000) | -+ base_addr = (entry_1 >> 16) | -+ (entry_2 & 0xff000000) | - ((entry_2 & 0xff) << 16); - target_ldt_info->base_addr = tswapal(base_addr); - target_ldt_info->limit = tswap32(limit); -@@ -4572,6 +4614,7 @@ +@@ -6354,6 +6358,7 @@ ret = fork(); if (ret == 0) { /* Child Process. */ + chatkey_close_fp(); - rcu_after_fork(); cpu_clone_regs(env, newsp); fork_end(1); -@@ -5561,6 +5604,7 @@ + /* There is a race condition here. The parent process could +@@ -7764,6 +7769,7 @@ #ifdef TARGET_GPROF _mcleanup(); #endif @@ -158,33 +27,7 @@ gdb_exit(cpu_env, arg1); _exit(arg1); ret = 0; /* avoid warning */ -@@ -5749,10 +5793,21 @@ - } - if (!(p = lock_user_string(arg1))) - goto execve_efault; -- ret = get_errno(execve(p, argp, envp)); -- unlock_user(p, arg1, 0); -- -- goto execve_end; -+ /* If execve() call is expected to successfully load and execute the -+ * specified program, call chatkey_exit() and abort, since the newly -+ * executed program is not out target. Also, if the new program is -+ * loaded and executed, it often ignores SIGTERM signal sent from -+ * the driver, and Chatkey falls into infinite loop. */ -+ if (check_executable(p) != 0) { -+ unlock_user(p, arg1, 0); -+ goto execve_efault; -+ } -+ chatkey_exit(); -+ exit(0); -+ //ret = get_errno(execve(p, argp, envp)); -+ //unlock_user(p, arg1, 0); -+ // -+ //goto execve_end; - - execve_efault: - ret = -TARGET_EFAULT; -@@ -7422,6 +7477,7 @@ +@@ -9820,6 +9826,7 @@ #ifdef TARGET_GPROF _mcleanup(); #endif @@ -192,66 +35,26 @@ gdb_exit(cpu_env, arg1); ret = get_errno(exit_group(arg1)); break; -@@ -8316,7 +8372,7 @@ +@@ -11688,8 +11695,20 @@ break; - #if defined(TARGET_NR_fchownat) - case TARGET_NR_fchownat: -- if (!(p = lock_user_string(arg2))) -+ if (!(p = lock_user_string(arg2))) - goto efault; - ret = get_errno(fchownat(arg1, p, low2highuid(arg3), - low2highgid(arg4), arg5)); -@@ -8801,7 +8857,7 @@ - case TARGET_F_GETLK64: - #ifdef TARGET_ARM - if (((CPUARMState *)cpu_env)->eabi) { -- if (!lock_user_struct(VERIFY_READ, target_efl, arg3, 1)) -+ if (!lock_user_struct(VERIFY_READ, target_efl, arg3, 1)) - goto efault; - fl.l_type = tswap16(target_efl->l_type); - fl.l_whence = tswap16(target_efl->l_whence); -@@ -8812,7 +8868,7 @@ - } else - #endif - { -- if (!lock_user_struct(VERIFY_READ, target_fl, arg3, 1)) -+ if (!lock_user_struct(VERIFY_READ, target_fl, arg3, 1)) - goto efault; - fl.l_type = tswap16(target_fl->l_type); - fl.l_whence = tswap16(target_fl->l_whence); -@@ -8825,7 +8881,7 @@ - if (ret == 0) { - #ifdef TARGET_ARM - if (((CPUARMState *)cpu_env)->eabi) { -- if (!lock_user_struct(VERIFY_WRITE, target_efl, arg3, 0)) -+ if (!lock_user_struct(VERIFY_WRITE, target_efl, arg3, 0)) - goto efault; - target_efl->l_type = tswap16(fl.l_type); - target_efl->l_whence = tswap16(fl.l_whence); -@@ -8836,7 +8892,7 @@ - } else - #endif - { -- if (!lock_user_struct(VERIFY_WRITE, target_fl, arg3, 0)) -+ if (!lock_user_struct(VERIFY_WRITE, target_fl, arg3, 0)) - goto efault; - target_fl->l_type = tswap16(fl.l_type); - target_fl->l_whence = tswap16(fl.l_whence); -@@ -8852,7 +8908,7 @@ - case TARGET_F_SETLKW64: - #ifdef TARGET_ARM - if (((CPUARMState *)cpu_env)->eabi) { -- if (!lock_user_struct(VERIFY_READ, target_efl, arg3, 1)) -+ if (!lock_user_struct(VERIFY_READ, target_efl, arg3, 1)) - goto efault; - fl.l_type = tswap16(target_efl->l_type); - fl.l_whence = tswap16(target_efl->l_whence); -@@ -8863,7 +8919,7 @@ - } else - #endif - { -- if (!lock_user_struct(VERIFY_READ, target_fl, arg3, 1)) -+ if (!lock_user_struct(VERIFY_READ, target_fl, arg3, 1)) - goto efault; - fl.l_type = tswap16(target_fl->l_type); - fl.l_whence = tswap16(target_fl->l_whence); + + case TARGET_NR_tgkill: +- ret = get_errno(safe_tgkill((int)arg1, (int)arg2, +- target_to_host_signal(arg3))); ++ { ++ int pid = (int)arg1, ++ tgid = (int)arg2, ++ sig = (int)arg3; ++ ++ /* Thought to be a workaround for -lpthread to make abort() work ++ * properly, without killing the forkserver due to a prematurely ++ * cached PID. */ ++ ++ if (afl_forksrv_pid && afl_forksrv_pid == pid && sig == SIGABRT) ++ pid = tgid = getpid(); ++ ++ ret = get_errno(safe_tgkill(pid, tgid, target_to_host_signal(sig))); ++ } + break; + + #ifdef TARGET_NR_set_robust_list diff --git a/Instrumentor/qemu/patches-coverage/translate.diff b/Instrumentor/qemu/patches-coverage/translate.diff deleted file mode 100644 index 3d5ffe2..0000000 --- a/Instrumentor/qemu/patches-coverage/translate.diff +++ /dev/null @@ -1,219 +0,0 @@ ---- qemu-2.3.0-coverage/target-i386/translate.c.orig 2017-10-07 02:23:34.787285527 +0900 -+++ qemu-2.3.0-coverage/target-i386/translate.c 2017-10-07 02:15:08.076137509 +0900 -@@ -391,7 +391,7 @@ - tcg_gen_addi_tl(cpu_A0, cpu_A0, val); - } - #endif -- -+ - static void gen_add_A0_im(DisasContext *s, int val) - { - #ifdef TARGET_X86_64 -@@ -1586,14 +1586,14 @@ - t0 = tcg_const_i32(0); - t1 = tcg_temp_new_i32(); - tcg_gen_trunc_tl_i32(t1, cpu_T[1]); -- tcg_gen_movi_i32(cpu_tmp2_i32, CC_OP_ADCOX); -+ tcg_gen_movi_i32(cpu_tmp2_i32, CC_OP_ADCOX); - tcg_gen_movi_i32(cpu_tmp3_i32, CC_OP_EFLAGS); - tcg_gen_movcond_i32(TCG_COND_NE, cpu_cc_op, t1, t0, - cpu_tmp2_i32, cpu_tmp3_i32); - tcg_temp_free_i32(t0); - tcg_temp_free_i32(t1); - -- /* The CC_OP value is no longer predictable. */ -+ /* The CC_OP value is no longer predictable. */ - set_cc_op(s, CC_OP_DYNAMIC); - } - -@@ -1686,7 +1686,7 @@ - gen_op_ld_v(s, ot, cpu_T[0], cpu_A0); - else - gen_op_mov_v_reg(ot, cpu_T[0], op1); -- -+ - if (is_right) { - switch (ot) { - case MO_8: -@@ -2282,17 +2282,17 @@ - - static inline void gen_op_movl_T0_seg(int seg_reg) - { -- tcg_gen_ld32u_tl(cpu_T[0], cpu_env, -+ tcg_gen_ld32u_tl(cpu_T[0], cpu_env, - offsetof(CPUX86State,segs[seg_reg].selector)); - } - - static inline void gen_op_movl_seg_T0_vm(int seg_reg) - { - tcg_gen_andi_tl(cpu_T[0], cpu_T[0], 0xffff); -- tcg_gen_st32_tl(cpu_T[0], cpu_env, -+ tcg_gen_st32_tl(cpu_T[0], cpu_env, - offsetof(CPUX86State,segs[seg_reg].selector)); - tcg_gen_shli_tl(cpu_T[0], cpu_T[0], 4); -- tcg_gen_st_tl(cpu_T[0], cpu_env, -+ tcg_gen_st_tl(cpu_T[0], cpu_env, - offsetof(CPUX86State,segs[seg_reg].base)); - } - -@@ -3089,7 +3089,7 @@ - #endif - { - gen_ldst_modrm(env, s, modrm, MO_32, OR_TMP0, 0); -- tcg_gen_addi_ptr(cpu_ptr0, cpu_env, -+ tcg_gen_addi_ptr(cpu_ptr0, cpu_env, - offsetof(CPUX86State,fpregs[reg].mmx)); - tcg_gen_trunc_tl_i32(cpu_tmp2_i32, cpu_T[0]); - gen_helper_movl_mm_T0_mmx(cpu_ptr0, cpu_tmp2_i32); -@@ -3099,14 +3099,14 @@ - #ifdef TARGET_X86_64 - if (s->dflag == MO_64) { - gen_ldst_modrm(env, s, modrm, MO_64, OR_TMP0, 0); -- tcg_gen_addi_ptr(cpu_ptr0, cpu_env, -+ tcg_gen_addi_ptr(cpu_ptr0, cpu_env, - offsetof(CPUX86State,xmm_regs[reg])); - gen_helper_movq_mm_T0_xmm(cpu_ptr0, cpu_T[0]); - } else - #endif - { - gen_ldst_modrm(env, s, modrm, MO_32, OR_TMP0, 0); -- tcg_gen_addi_ptr(cpu_ptr0, cpu_env, -+ tcg_gen_addi_ptr(cpu_ptr0, cpu_env, - offsetof(CPUX86State,xmm_regs[reg])); - tcg_gen_trunc_tl_i32(cpu_tmp2_i32, cpu_T[0]); - gen_helper_movl_mm_T0_xmm(cpu_ptr0, cpu_tmp2_i32); -@@ -3263,13 +3263,13 @@ - case 0x7e: /* movd ea, mm */ - #ifdef TARGET_X86_64 - if (s->dflag == MO_64) { -- tcg_gen_ld_i64(cpu_T[0], cpu_env, -+ tcg_gen_ld_i64(cpu_T[0], cpu_env, - offsetof(CPUX86State,fpregs[reg].mmx)); - gen_ldst_modrm(env, s, modrm, MO_64, OR_TMP0, 1); - } else - #endif - { -- tcg_gen_ld32u_tl(cpu_T[0], cpu_env, -+ tcg_gen_ld32u_tl(cpu_T[0], cpu_env, - offsetof(CPUX86State,fpregs[reg].mmx.MMX_L(0))); - gen_ldst_modrm(env, s, modrm, MO_32, OR_TMP0, 1); - } -@@ -3277,13 +3277,13 @@ - case 0x17e: /* movd ea, xmm */ - #ifdef TARGET_X86_64 - if (s->dflag == MO_64) { -- tcg_gen_ld_i64(cpu_T[0], cpu_env, -+ tcg_gen_ld_i64(cpu_T[0], cpu_env, - offsetof(CPUX86State,xmm_regs[reg].XMM_Q(0))); - gen_ldst_modrm(env, s, modrm, MO_64, OR_TMP0, 1); - } else - #endif - { -- tcg_gen_ld32u_tl(cpu_T[0], cpu_env, -+ tcg_gen_ld32u_tl(cpu_T[0], cpu_env, - offsetof(CPUX86State,xmm_regs[reg].XMM_L(0))); - gen_ldst_modrm(env, s, modrm, MO_32, OR_TMP0, 1); - } -@@ -3408,14 +3408,14 @@ - break; - case 0x050: /* movmskps */ - rm = (modrm & 7) | REX_B(s); -- tcg_gen_addi_ptr(cpu_ptr0, cpu_env, -+ tcg_gen_addi_ptr(cpu_ptr0, cpu_env, - offsetof(CPUX86State,xmm_regs[rm])); - gen_helper_movmskps(cpu_tmp2_i32, cpu_env, cpu_ptr0); - tcg_gen_extu_i32_tl(cpu_regs[reg], cpu_tmp2_i32); - break; - case 0x150: /* movmskpd */ - rm = (modrm & 7) | REX_B(s); -- tcg_gen_addi_ptr(cpu_ptr0, cpu_env, -+ tcg_gen_addi_ptr(cpu_ptr0, cpu_env, - offsetof(CPUX86State,xmm_regs[rm])); - gen_helper_movmskpd(cpu_tmp2_i32, cpu_env, cpu_ptr0); - tcg_gen_extu_i32_tl(cpu_regs[reg], cpu_tmp2_i32); -@@ -4945,11 +4945,13 @@ - gen_push_v(s, cpu_T[1]); - gen_op_jmp_v(cpu_T[0]); - gen_eob(s); -+ s->tb->is_call = 1; - break; - case 3: /* lcall Ev */ - gen_op_ld_v(s, ot, cpu_T[1], cpu_A0); - gen_add_A0_im(s, 1 << ot); - gen_op_ld_v(s, MO_16, cpu_T[0], cpu_A0); -+ s->tb->is_call = 1; - do_lcall: - if (s->pe && !s->vm86) { - gen_update_cc_op(s); -@@ -5217,7 +5219,7 @@ - gen_lea_modrm(env, s, modrm); - gen_helper_cmpxchg16b(cpu_env, cpu_A0); - } else --#endif -+#endif - { - if (!(s->cpuid_features & CPUID_CX8)) - goto illegal_op; -@@ -6276,7 +6278,7 @@ - case 0x6d: - ot = mo_b_d32(b, dflag); - tcg_gen_ext16u_tl(cpu_T[0], cpu_regs[R_EDX]); -- gen_check_io(s, ot, pc_start - s->cs_base, -+ gen_check_io(s, ot, pc_start - s->cs_base, - SVM_IOIO_TYPE_MASK | svm_is_rep(prefixes) | 4); - if (prefixes & (PREFIX_REPZ | PREFIX_REPNZ)) { - gen_repz_ins(s, ot, pc_start - s->cs_base, s->pc - s->cs_base); -@@ -6398,6 +6400,7 @@ - /* Note that gen_pop_T0 uses a zero-extending load. */ - gen_op_jmp_v(cpu_T[0]); - gen_eob(s); -+ s->tb->is_ret = 1; - break; - case 0xca: /* lret im */ - val = cpu_ldsw_code(env, s->pc); -@@ -6466,6 +6469,7 @@ - tcg_gen_movi_tl(cpu_T[0], next_eip); - gen_push_v(s, cpu_T[0]); - gen_jmp(s, tval); -+ s->tb->is_call = 1; - } - break; - case 0x9a: /* lcall im */ -@@ -6481,6 +6485,7 @@ - tcg_gen_movi_tl(cpu_T[0], selector); - tcg_gen_movi_tl(cpu_T[1], offset); - } -+ s->tb->is_call = 1; - goto do_lcall; - case 0xe9: /* jmp im */ - if (dflag != MO_16) { -@@ -7338,7 +7343,7 @@ - break; - case 4: /* STGI */ - if ((!(s->flags & HF_SVME_MASK) && -- !(s->cpuid_ext3_features & CPUID_EXT3_SKINIT)) || -+ !(s->cpuid_ext3_features & CPUID_EXT3_SKINIT)) || - !s->pe) - goto illegal_op; - if (s->cpl != 0) { -@@ -7359,8 +7364,8 @@ - } - break; - case 6: /* SKINIT */ -- if ((!(s->flags & HF_SVME_MASK) && -- !(s->cpuid_ext3_features & CPUID_EXT3_SKINIT)) || -+ if ((!(s->flags & HF_SVME_MASK) && -+ !(s->cpuid_ext3_features & CPUID_EXT3_SKINIT)) || - !s->pe) - goto illegal_op; - gen_helper_skinit(cpu_env); -@@ -7938,6 +7943,9 @@ - dc->cc_op = CC_OP_DYNAMIC; - dc->cc_op_dirty = false; - dc->cs_base = cs_base; -+ /* Chatkey : initialize 'is_call', 'is_ret' flags */ -+ tb->is_call = 0; -+ tb->is_ret = 0; - dc->tb = tb; - dc->popl_esp_hack = 0; - /* select memory access functions */ diff --git a/Instrumentor/qemu/prepare_qemu.sh b/Instrumentor/qemu/prepare_qemu.sh index ab07b61..35b0ff2 100755 --- a/Instrumentor/qemu/prepare_qemu.sh +++ b/Instrumentor/qemu/prepare_qemu.sh @@ -4,7 +4,7 @@ # # Modified codes from AFL's QEMU mode (original license below). # -------------------------------------- -# + # Written by Andrew Griffiths and # Michal Zalewski # @@ -16,8 +16,9 @@ # # http://www.apache.org/licenses/LICENSE-2.0 -QEMU_URL="https://download.qemu.org/qemu-2.3.0.tar.bz2" -QEMU_SHA384="7a0f0c900f7e2048463cc32ff3e904965ab466c8428847400a0f2dcfe458108a68012c4fddb2a7e7c822b4fd1a49639b" +VERSION="2.10.0" +QEMU_URL="https://download.qemu.org/qemu-${VERSION}.tar.bz2" +QEMU_SHA384="9496d1d209d3a49d67dd83fbcf3f2bf376b7d5f500b0247d813639d104effa79d8a39e64f94d6f9541f6e9d8e3cc574f" echo "=========================================" echo "Chatkey instrumentation QEMU build script" @@ -33,7 +34,7 @@ if [ ! "`uname -s`" = "Linux" ]; then fi -if [ ! -f "patches-coverage/chatkey.cc" -o ! -f "patches-branch/chatkey.c" ]; then +if [ ! -f "patches-coverage/chatkey.c" -o ! -f "patches-branch/chatkey.c" ]; then echo "[-] Error: key files not found - wrong working directory?" exit 1 @@ -68,7 +69,7 @@ CKSUM=`sha384sum -- "$ARCHIVE" 2>/dev/null | cut -d' ' -f1` if [ ! "$CKSUM" = "$QEMU_SHA384" ]; then - echo "[*] Downloading QEMU 2.3.0 from the web..." + echo "[*] Downloading QEMU from the web..." rm -f "$ARCHIVE" wget -O "$ARCHIVE" -- "$QEMU_URL" || exit 1 @@ -87,96 +88,86 @@ else fi -echo "[*] Uncompressing archive (this will take a while)..." - -rm -rf "qemu-2.3.0" || exit 1 -rm -rf "qemu-2.3.0-coverage" || exit 1 -rm -rf "qemu-2.3.0-branch" || exit 1 -rm -rf "qemu-2.3.0-bbcount" || exit 1 -rm -rf "qemu-2.3.0-coverage-x86" || exit 1 -rm -rf "qemu-2.3.0-coverage-x64" || exit 1 -rm -rf "qemu-2.3.0-branch-x86" || exit 1 -rm -rf "qemu-2.3.0-branch-x64" || exit 1 -rm -rf "qemu-2.3.0-bbcount-x86" || exit 1 -rm -rf "qemu-2.3.0-bbcount-x64" || exit 1 +echo "[*] Clean up directories..." + +rm -rf "qemu-${VERSION}" || exit 1 +rm -rf "qemu-${VERSION}-coverage" || exit 1 +rm -rf "qemu-${VERSION}-branch" || exit 1 +rm -rf "qemu-${VERSION}-bbcount" || exit 1 +rm -rf "qemu-${VERSION}-coverage-x86" || exit 1 +rm -rf "qemu-${VERSION}-coverage-x64" || exit 1 +rm -rf "qemu-${VERSION}-branch-x86" || exit 1 +rm -rf "qemu-${VERSION}-branch-x64" || exit 1 +rm -rf "qemu-${VERSION}-bbcount-x86" || exit 1 +rm -rf "qemu-${VERSION}-bbcount-x64" || exit 1 + +echo "[*] Uncompressing archive..." + tar xf "$ARCHIVE" || exit 1 echo "[+] Unpacking successful." echo "[*] Backup target files of patches-common/ (for later use)" -cp qemu-2.3.0/linux-user/elfload.c qemu-2.3.0/linux-user/elfload.c.orig -cp qemu-2.3.0/linux-user/linuxload.c qemu-2.3.0/linux-user/linuxload.c.orig -cp qemu-2.3.0/linux-user/signal.c qemu-2.3.0/linux-user/signal.c.orig -cp qemu-2.3.0/translate-all.c qemu-2.3.0/translate-all.c.orig -cp qemu-2.3.0/scripts/texi2pod.pl qemu-2.3.0/scripts/texi2pod.pl.orig -cp qemu-2.3.0/user-exec.c qemu-2.3.0/user-exec.c.orig -cp qemu-2.3.0/configure qemu-2.3.0/configure.orig -cp qemu-2.3.0/include/sysemu/os-posix.h qemu-2.3.0/include/sysemu/os-posix.h.orig +cp qemu-${VERSION}/configure qemu-${VERSION}/configure.orig +cp qemu-${VERSION}/linux-user/elfload.c qemu-${VERSION}/linux-user/elfload.c.orig +cp qemu-${VERSION}/util/memfd.c qemu-${VERSION}/util/memfd.c.orig +cp qemu-${VERSION}/linux-user/signal.c qemu-${VERSION}/linux-user/signal.c.orig echo "[*] Applying common patches..." +patch -p0 and -# Michal Zalewski -# -# Copyright 2015, 2016 Google Inc. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at: -# -# http://www.apache.org/licenses/LICENSE-2.0 -# - -build_qemu () { - if [ $2 = "x86" ]; then - CPU_TARGET="i386" - elif [ $2 = "x64" ]; then - CPU_TARGET="x86_64" - else - echo "Invalid CPU architecture provided" - exit 0 - fi - - echo "[*] Configuring QEMU for $CPU_TARGET..." - - cd qemu-2.3.0-$1-$2 || exit 1 - - CFLAGS="-O3" ./configure --disable-system --enable-linux-user \ - --enable-guest-base --disable-gtk --disable-sdl --disable-vnc \ - --target-list="${CPU_TARGET}-linux-user" || exit 1 - - echo "[+] Configuration complete." - - echo "[*] Attempting to build QEMU (fingers crossed!)..." - - make || exit 1 - - echo "[+] Build process successful!" - - echo "[*] Copying binary..." - cp -f "${CPU_TARGET}-linux-user/qemu-${CPU_TARGET}" "../qemu-trace" || exit 1 - cd .. -} - -echo "=========================================" -echo "Chatkey instrumentation QEMU build script" -echo "=========================================" - - -echo "[+] Sanity checking and patching omitted." - -build_qemu coverage x86 -mv "./qemu-trace" "../../build/qemu-trace-coverage-x86" || exit 1 -echo "[+] Successfully created 'qemu-trace-coverage-x86'." - -build_qemu coverage x64 -mv "./qemu-trace" "../../build/qemu-trace-coverage-x64" || exit 1 -echo "[+] Successfully created 'qemu-trace-coverage-x64'." - -build_qemu branch x86 -mv "./qemu-trace" "../../build/qemu-trace-branch-x86" || exit 1 -echo "[+] Successfully created 'qemu-trace-branch-x86'." - -build_qemu branch x64 -mv "./qemu-trace" "../../build/qemu-trace-branch-x64" || exit 1 -echo "[+] Successfully created 'qemu-trace-branch-x64'." - -build_qemu bbcount x86 -mv "./qemu-trace" "../../build/qemu-trace-bbcount-x86" || exit 1 -echo "[+] Successfully created 'qemu-trace-bbcount-x86'." - -build_qemu bbcount x64 -mv "./qemu-trace" "../../build/qemu-trace-bbcount-x64" || exit 1 -echo "[+] Successfully created 'qemu-trace-bbcount-x64'." - -exit 0 diff --git a/Instrumentor/qemu/repatch.sh b/Instrumentor/qemu/repatch.sh index d58ecac..a259683 100755 --- a/Instrumentor/qemu/repatch.sh +++ b/Instrumentor/qemu/repatch.sh @@ -1,95 +1,64 @@ #!/bin/sh # -# Chatkey QEMU build script -# -# Modified codes are Copyright 2016 KAIST SoftSec. -# -# Copied from afl-fuzz QEMU mode -# -------------------------------------- -# -# Written by Andrew Griffiths and -# Michal Zalewski -# -# Copyright 2015, 2016 Google Inc. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at: -# -# http://www.apache.org/licenses/LICENSE-2.0 -# +# QEMU rebuild script for Eclipser's instrumentation + +VERSION="2.10.0" ##### Common patch export_common_patch() { - TARG_DIR=./qemu-2.3.0-$1-$2 - cp qemu-2.3.0/linux-user/elfload.c ./$TARG_DIR/linux-user/ - cp qemu-2.3.0/linux-user/linuxload.c ./$TARG_DIR/linux-user/ - cp qemu-2.3.0/linux-user/signal.c ./$TARG_DIR/linux-user/ - cp qemu-2.3.0/translate-all.c ./$TARG_DIR/ - cp qemu-2.3.0/scripts/texi2pod.pl ./$TARG_DIR/scripts/ - cp qemu-2.3.0/user-exec.c ./$TARG_DIR/ - cp qemu-2.3.0/configure ./$TARG_DIR/ - cp qemu-2.3.0/include/sysemu/os-posix.h ./$TARG_DIR/include/sysemu/ + TARG_DIR=./qemu-${VERSION}-$1-$2 + cp qemu-${VERSION}/configure ./$TARG_DIR/ + cp qemu-${VERSION}/linux-user/elfload.c ./$TARG_DIR/linux-user/ + cp qemu-${VERSION}/util/memfd.c ./$TARG_DIR/util/ + cp qemu-${VERSION}/linux-user/signal.c ./$TARG_DIR/linux-user/ } export_coverage_patch() { - TARG_DIR=./qemu-2.3.0-coverage-$1 - cp qemu-2.3.0-coverage/linux-user/syscall.c $TARG_DIR/linux-user/syscall.c - cp qemu-2.3.0-coverage/cpu-exec.c $TARG_DIR/cpu-exec.c - cp qemu-2.3.0-coverage/target-i386/translate.c $TARG_DIR/target-i386/translate.c - cp qemu-2.3.0-coverage/include/exec/exec-all.h ./$TARG_DIR/include/exec/exec-all.h - cp qemu-2.3.0-coverage/Makefile.target $TARG_DIR/Makefile.target - cp qemu-2.3.0-coverage/chatkey.cc $TARG_DIR/ - cp qemu-2.3.0-coverage/afl-qemu-cpu-inl.h $TARG_DIR/ + TARG_DIR=./qemu-${VERSION}-coverage-$1 + cp qemu-${VERSION}-coverage/accel/tcg/afl-qemu-cpu-inl.h $TARG_DIR/accel/tcg/ + cp qemu-${VERSION}-coverage/accel/tcg/chatkey.c $TARG_DIR/accel/tcg/ + cp qemu-${VERSION}-coverage/accel/tcg/cpu-exec.c $TARG_DIR/accel/tcg/cpu-exec.c + cp qemu-${VERSION}-coverage/accel/tcg/Makefile.objs $TARG_DIR/accel/tcg/Makefile.objs + cp qemu-${VERSION}-coverage/linux-user/syscall.c $TARG_DIR/linux-user/syscall.c } export_branch_patch() { - TARG_DIR=./qemu-2.3.0-branch-$1 - cp qemu-2.3.0-branch/cpu-exec.c $TARG_DIR/cpu-exec.c - cp qemu-2.3.0-branch/linux-user/syscall.c $TARG_DIR/linux-user/syscall.c - cp qemu-2.3.0-branch/Makefile.target $TARG_DIR/Makefile.target - cp qemu-2.3.0-branch/target-i386/translate.c $TARG_DIR/target-i386/translate.c - cp qemu-2.3.0-branch/tcg/i386/tcg-target.c $TARG_DIR/tcg/i386/tcg-target.c - cp qemu-2.3.0-branch/tcg/tcg-op.h $TARG_DIR/tcg/tcg-op.h - cp qemu-2.3.0-branch/tcg/tcg-opc.h $TARG_DIR/tcg/tcg-opc.h - cp qemu-2.3.0-branch/tcg/tcg.h $TARG_DIR/tcg/tcg.h - cp qemu-2.3.0-branch/tcg/optimize.c $TARG_DIR/tcg/optimize.c - cp qemu-2.3.0-branch/tcg/chatkey.c $TARG_DIR/tcg/ - cp qemu-2.3.0-branch/afl-qemu-cpu-inl.h $TARG_DIR/ + TARG_DIR=./qemu-${VERSION}-branch-$1 + cp qemu-${VERSION}-branch/afl-qemu-cpu-inl.h $TARG_DIR/ + cp qemu-${VERSION}-branch/tcg/chatkey.c $TARG_DIR/tcg/ + cp qemu-${VERSION}-branch/accel/tcg/cpu-exec.c $TARG_DIR/accel/tcg/cpu-exec.c + cp qemu-${VERSION}-branch/Makefile.target $TARG_DIR/Makefile.target + cp qemu-${VERSION}-branch/linux-user/syscall.c $TARG_DIR/linux-user/syscall.c + + cp qemu-${VERSION}-branch/tcg/optimize.c $TARG_DIR/tcg/optimize.c + cp qemu-${VERSION}-branch/tcg/tcg-op.h $TARG_DIR/tcg/tcg-op.h + cp qemu-${VERSION}-branch/tcg/tcg-opc.h $TARG_DIR/tcg/tcg-opc.h + cp qemu-${VERSION}-branch/tcg/i386/tcg-target.inc.c $TARG_DIR/tcg/i386/tcg-target.inc.c + cp qemu-${VERSION}-branch/target/i386/translate.c $TARG_DIR/target/i386/translate.c } export_bbcount_patch() { - TARG_DIR=./qemu-2.3.0-bbcount-$1 - cp qemu-2.3.0-bbcount/linux-user/syscall.c $TARG_DIR/linux-user/syscall.c - cp qemu-2.3.0-bbcount/cpu-exec.c $TARG_DIR/cpu-exec.c - cp qemu-2.3.0-bbcount/Makefile.target $TARG_DIR/Makefile.target - cp qemu-2.3.0-bbcount/linux-user/chatkey.cc $TARG_DIR/linux-user/ - cp qemu-2.3.0-bbcount/linux-user/Makefile.objs $TARG_DIR/linux-user/Makefile.objs - cp qemu-2.3.0-bbcount/linux-user/main.c $TARG_DIR/linux-user/main.c + TARG_DIR=./qemu-${VERSION}-bbcount-$1 + cp qemu-${VERSION}-bbcount/chatkey.cc $TARG_DIR/ + cp qemu-${VERSION}-bbcount/accel/tcg/cpu-exec.c $TARG_DIR/accel/tcg/cpu-exec.c + cp qemu-${VERSION}-bbcount/Makefile.target $TARG_DIR/Makefile.target + cp qemu-${VERSION}-bbcount/linux-user/syscall.c $TARG_DIR/linux-user/syscall.c } ##### Common patch -# Recover -cp qemu-2.3.0/linux-user/elfload.c.orig qemu-2.3.0/linux-user/elfload.c -cp qemu-2.3.0/linux-user/linuxload.c.orig qemu-2.3.0/linux-user/linuxload.c -cp qemu-2.3.0/linux-user/signal.c.orig qemu-2.3.0/linux-user/signal.c -cp qemu-2.3.0/translate-all.c.orig qemu-2.3.0/translate-all.c -cp qemu-2.3.0/scripts/texi2pod.pl.orig qemu-2.3.0/scripts/texi2pod.pl -cp qemu-2.3.0/user-exec.c.orig qemu-2.3.0/user-exec.c -cp qemu-2.3.0/configure.orig qemu-2.3.0/configure -cp qemu-2.3.0/include/sysemu/os-posix.h.orig qemu-2.3.0/include/sysemu/os-posix.h +# Recover original files. +cp qemu-${VERSION}/configure.orig qemu-${VERSION}/configure +cp qemu-${VERSION}/linux-user/elfload.c.orig qemu-${VERSION}/linux-user/elfload.c +cp qemu-${VERSION}/util/memfd.c.orig qemu-${VERSION}/util/memfd.c +cp qemu-${VERSION}/linux-user/signal.c.orig qemu-${VERSION}/linux-user/signal.c # Patch +patch -p0 Date: Sun, 4 Oct 2020 22:21:30 +0900 Subject: [PATCH 13/29] Reinvestigate by-products of branch trace collection If we observe a coverage gain while collecting branch traces, evaluate those candidate seeds again after performing grey-box concolic testing. This is to prefer the grey-box concolic testing solution seeds over by-product seeds. To implement this, introduce a non-cumulative coverage measurement mode to QEMU tracer. --- Instrumentor/qemu/patches-branch/chatkey.c | 9 ++++-- src/Core/Executor.fs | 34 +++++++++++----------- src/Core/Typedef.fs | 12 ++++++++ src/GreyConcolic/BranchTrace.fs | 12 ++++++-- src/GreyConcolic/GreyConcolic.fs | 10 ++++++- 5 files changed, 54 insertions(+), 23 deletions(-) diff --git a/Instrumentor/qemu/patches-branch/chatkey.c b/Instrumentor/qemu/patches-branch/chatkey.c index 52c23c0..14ea56e 100644 --- a/Instrumentor/qemu/patches-branch/chatkey.c +++ b/Instrumentor/qemu/patches-branch/chatkey.c @@ -24,6 +24,9 @@ extern unsigned int afl_forksrv_pid; #define MAX_TRACE_LEN (1000000) +#define IGNORE_COVERAGE 1 +#define NOCMULATIVE_COVERAGE 2 +#define CUMULATIVE_COVERAGE 3 void flush_trace_buffer(void); void chatkey_setup_before_forkserver(void); @@ -82,7 +85,7 @@ void chatkey_setup_after_forkserver(void) { measure_coverage = atoi(getenv("CK_MEASURE_COV")); } - if (measure_coverage == 1) { + if (measure_coverage != IGNORE_COVERAGE) { coverage_fp = fopen(coverage_path, "w"); assert(coverage_fp != NULL); } @@ -309,6 +312,8 @@ void chatkey_log_bb(abi_ulong addr) { new_byte = old_byte | byte_mask; if (old_byte != new_byte) { found_new_edge = 1; - edge_bitmap[byte_idx] = new_byte; + if (measure_coverage == CUMULATIVE_COVERAGE) { + edge_bitmap[byte_idx] = new_byte; + } } } diff --git a/src/Core/Executor.fs b/src/Core/Executor.fs index ec8ff8c..ae069e5 100644 --- a/src/Core/Executor.fs +++ b/src/Core/Executor.fs @@ -62,12 +62,10 @@ let initialize opt = set_env("CK_BITMAP_LOG", System.IO.Path.GetFullPath(bitmapLog)) if verbosity >= 2 then set_env("CK_DBG_LOG", System.IO.Path.GetFullPath(dbgLog)) - set_env("CK_FEED_ADDR", "0") - set_env("CK_FEED_IDX", "0") - set_env("CK_MEASURE_COV", "0") + // Initialize C wrapper code. + initialize_exec () // Disable TranslationBlock chaining feature of QEMU. set_env("QEMU_LOG", "nochain") - initialize_exec () // Initialize fork server. forkServerEnabled <- true set_env("CK_FORK_SERVER", "1") @@ -190,18 +188,18 @@ let private runCoverageTracerForked opt stdin = if signal = Signal.ERROR then abandonForkServer () signal -let private runBranchTracerForked opt stdin addr idx measureCov = +let private runBranchTracerForked opt stdin addr idx covMeasure = let timeout = opt.ExecTimeout let stdLen = Array.length stdin - let covFlag = if measureCov then 1 else 0 - let signal = exec_fork_branch(timeout, stdLen, stdin, addr, idx, covFlag) + let covEnum = CoverageMeasure.toEnum covMeasure + let signal = exec_fork_branch(timeout, stdLen, stdin, addr, idx, covEnum) if signal = Signal.ERROR then abandonForkServer () signal -let private setEnvForBranch (addr: uint64) (idx: uint32) measureCov = +let private setEnvForBranch (addr: uint64) (idx: uint32) covMeasure = set_env("CK_FEED_ADDR", sprintf "%016x" addr) set_env("CK_FEED_IDX", sprintf "%016x" idx) - set_env("CK_MEASURE_COV", sprintf "%d" (if measureCov then 1 else 0)) + set_env("CK_MEASURE_COV", sprintf "%d" (CoverageMeasure.toEnum covMeasure)) (*** Top-level tracer executor functions ***) @@ -216,9 +214,10 @@ let getCoverage opt seed = let getBranchTrace opt seed tryVal = setupFile seed let stdin = prepareStdIn seed - let exitSig = - if forkServerEnabled then runBranchTracerForked opt stdin 0UL 0ul true - else setEnvForBranch 0UL 0ul true; runTracer Branch opt stdin + let exitSig = if forkServerEnabled + then runBranchTracerForked opt stdin 0UL 0ul NonCumulative + else setEnvForBranch 0UL 0ul NonCumulative + runTracer Branch opt stdin let coverageGain = parseCoverage coverageLog let branchTrace = readBranchTrace opt branchLog tryVal removeFile coverageLog @@ -228,9 +227,10 @@ let getBranchInfo opt seed tryVal targPoint = setupFile seed let stdin = prepareStdIn seed let addr, idx = targPoint.Addr, uint32 targPoint.Idx - let exitSig = - if forkServerEnabled then runBranchTracerForked opt stdin addr idx true - else setEnvForBranch addr idx true; runTracer Branch opt stdin + let exitSig = if forkServerEnabled + then runBranchTracerForked opt stdin addr idx Cumulative + else setEnvForBranch addr idx Cumulative + runTracer Branch opt stdin let coverageGain = parseCoverage coverageLog let branchInfoOpt = tryReadBranchInfo opt branchLog tryVal removeFile coverageLog @@ -240,8 +240,8 @@ let getBranchInfoOnly opt seed tryVal targPoint = setupFile seed let stdin = prepareStdIn seed let addr, idx = targPoint.Addr, uint32 targPoint.Idx - if forkServerEnabled then runBranchTracerForked opt stdin addr idx false - else setEnvForBranch addr idx false; runTracer Branch opt stdin + if forkServerEnabled then runBranchTracerForked opt stdin addr idx Ignore + else setEnvForBranch addr idx Ignore; runTracer Branch opt stdin |> ignore let brInfoOpt = tryReadBranchInfo opt branchLog tryVal brInfoOpt diff --git a/src/Core/Typedef.fs b/src/Core/Typedef.fs index 414d935..9fe8093 100644 --- a/src/Core/Typedef.fs +++ b/src/Core/Typedef.fs @@ -13,6 +13,18 @@ type InputSource = /// Fuzz file input source. | FileInput of filepath: string +/// Describes how to measure coverage in QEMU tracer for branch feedback. +type CoverageMeasure = + | Ignore // Just collect branch information and ignore coverage. + | NonCumulative // Measure coverage as well, but without any side effect. + | Cumulative // Measure coverage as well, in cumulative manner. + +module CoverageMeasure = + let toEnum = function + | Ignore -> 1 + | NonCumulative -> 2 + | Cumulative -> 3 + /// Describes the gain of coverage. type CoverageGain = | NoGain diff --git a/src/GreyConcolic/BranchTrace.fs b/src/GreyConcolic/BranchTrace.fs index b7feffa..1fe2516 100644 --- a/src/GreyConcolic/BranchTrace.fs +++ b/src/GreyConcolic/BranchTrace.fs @@ -7,16 +7,22 @@ type BranchTrace = BranchInfo list module BranchTrace = - let collectAux opt seed tryVal = + let collectAux opt seed acc tryVal = + let accTraces, accCandidates = acc let tryByteVal = Sampled (byte tryVal) let trySeed = Seed.updateCurByte seed tryByteVal let exitSig, covGain, trace = Executor.getBranchTrace opt trySeed tryVal - (trace, (trySeed, exitSig, covGain)) + let accTraces = trace :: accTraces + let accCandidates = if covGain = NewEdge || Signal.isCrash exitSig + then seed :: accCandidates + else accCandidates + (accTraces, accCandidates) let collect seed opt minVal maxVal = let nSpawn = opt.NSpawn let tryVals = sampleInt minVal maxVal nSpawn - List.map (collectAux opt seed) tryVals |> List.unzip + let traces, candidates = List.fold(collectAux opt seed) ([], []) tryVals + List.rev traces, List.rev candidates // To preserver order. let getHeadAddr (brTrace: BranchTrace) = match brTrace with diff --git a/src/GreyConcolic/GreyConcolic.fs b/src/GreyConcolic/GreyConcolic.fs index 4974964..1313c25 100644 --- a/src/GreyConcolic/GreyConcolic.fs +++ b/src/GreyConcolic/GreyConcolic.fs @@ -1,5 +1,12 @@ module Eclipser.GreyConcolic +// Evaluate seeds that gained coverage while collecting branch traces. If the +// newly covered code is also reached with grey-box concolic testing solutions, +// the coverage gain will disappear in this reinvestigation. +let private reconsiderCandidates opt seeds = + let sigs, covGains = List.map (Executor.getCoverage opt) seeds |> List.unzip + List.zip3 seeds sigs covGains + let run seed opt = let curByteVal = Seed.getCurByteVal seed let minByte, maxByte = ByteVal.getMinMax curByteVal seed.Source @@ -7,7 +14,7 @@ let run seed opt = let seedStr = Seed.toString seed failwithf "Cursor pointing to Fixed ByteVal %s" seedStr let minVal, maxVal = bigint (int minByte), bigint (int maxByte) - let branchTraces, byProducts = BranchTrace.collect seed opt minVal maxVal + let branchTraces, candidates = BranchTrace.collect seed opt minVal maxVal let byteDir = Seed.getByteCursorDir seed let bytes = Seed.queryNeighborBytes seed byteDir let ctx = { Bytes = bytes; ByteDir = byteDir } @@ -15,4 +22,5 @@ let run seed opt = let branchTree = BranchTree.selectAndRepair opt branchTree GreySolver.clearSolutionCache () let solutions = GreySolver.solve seed opt byteDir branchTree + let byProducts = reconsiderCandidates opt candidates solutions @ byProducts From d8bb95afd0bcd5be8cb7aef0e78d02f8ca3ea979 Mon Sep 17 00:00:00 2001 From: Jaeseung Choi Date: Sun, 4 Oct 2020 07:34:38 -0700 Subject: [PATCH 14/29] Reduce maximum branch trace length --- Instrumentor/qemu/patches-branch/chatkey.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Instrumentor/qemu/patches-branch/chatkey.c b/Instrumentor/qemu/patches-branch/chatkey.c index 14ea56e..88713f6 100644 --- a/Instrumentor/qemu/patches-branch/chatkey.c +++ b/Instrumentor/qemu/patches-branch/chatkey.c @@ -22,7 +22,7 @@ extern unsigned int afl_forksrv_pid; #define FORKSRV_FD 198 #define TSL_FD (FORKSRV_FD - 1) -#define MAX_TRACE_LEN (1000000) +#define MAX_TRACE_LEN (100000) #define IGNORE_COVERAGE 1 #define NOCMULATIVE_COVERAGE 2 From 30464ff9672b3ed0bdd020d4729a4946ec1b5c0a Mon Sep 17 00:00:00 2001 From: Jaeseung Choi Date: Sun, 4 Oct 2020 07:36:45 -0700 Subject: [PATCH 15/29] Fix misc typo in a test example --- examples/monoton.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/monoton.c b/examples/monoton.c index ceeef03..e9d2547 100644 --- a/examples/monoton.c +++ b/examples/monoton.c @@ -35,7 +35,7 @@ int main(int argc, char ** argv) { ((j << 8) & 0xff0000) | // move byte 1 to byte 2 ((j << 24) & 0xff000000); // byte 0 to byte 3 if (j * j == 0x10A29504) // 0x4142 ^ 2 - printf("Found new path 2\n"); + printf("Found new path 2!\n"); n = read(fd, buf, 8); buf[n] = '\0'; From c03f261196f25ad7da15d3d93df4e1a9807b4933 Mon Sep 17 00:00:00 2001 From: Jaeseung Choi Date: Sun, 4 Oct 2020 23:08:08 -0700 Subject: [PATCH 16/29] Cleanup the directory structure of instrumentation code --- .gitignore | 10 +++++----- Instrumentor/{qemu => }/build_qemu_x64.sh | 6 +++--- Instrumentor/{qemu => }/build_qemu_x86.sh | 6 +++--- Instrumentor/{qemu => }/generate_bbcount_patch.sh | 0 Instrumentor/{qemu => }/generate_branch_patch.sh | 0 Instrumentor/{qemu => }/generate_common_patch.sh | 0 Instrumentor/{qemu => }/generate_coverage_patch.sh | 0 Instrumentor/{qemu => }/patches-bbcount/chatkey.cc | 0 Instrumentor/{qemu => }/patches-bbcount/cpu-exec.diff | 0 .../{qemu => }/patches-bbcount/makefile-target.diff | 0 Instrumentor/{qemu => }/patches-bbcount/syscall.diff | 0 .../{qemu => }/patches-branch/afl-qemu-cpu-inl.h | 0 Instrumentor/{qemu => }/patches-branch/chatkey.c | 0 Instrumentor/{qemu => }/patches-branch/cpu-exec.diff | 0 .../{qemu => }/patches-branch/makefile-target.diff | 0 Instrumentor/{qemu => }/patches-branch/optimize.diff | 0 Instrumentor/{qemu => }/patches-branch/syscall.diff | 0 Instrumentor/{qemu => }/patches-branch/tcg-op.diff | 0 Instrumentor/{qemu => }/patches-branch/tcg-opc.diff | 0 Instrumentor/{qemu => }/patches-branch/tcg-target.diff | 0 Instrumentor/{qemu => }/patches-branch/tcg.diff | 0 Instrumentor/{qemu => }/patches-branch/translate.diff | 0 Instrumentor/{qemu => }/patches-common/configure.diff | 0 Instrumentor/{qemu => }/patches-common/elfload.diff | 0 Instrumentor/{qemu => }/patches-common/memfd.diff | 0 Instrumentor/{qemu => }/patches-common/signal.diff | 0 .../{qemu => }/patches-coverage/afl-qemu-cpu-inl.h | 0 Instrumentor/{qemu => }/patches-coverage/chatkey.c | 0 Instrumentor/{qemu => }/patches-coverage/cpu-exec.diff | 0 .../{qemu => }/patches-coverage/makefile-objs.diff | 0 Instrumentor/{qemu => }/patches-coverage/syscall.diff | 0 Instrumentor/{qemu => }/prepare_qemu.sh | 0 Instrumentor/{qemu => }/repatch.sh | 0 Makefile | 7 ++++--- 34 files changed, 15 insertions(+), 14 deletions(-) rename Instrumentor/{qemu => }/build_qemu_x64.sh (87%) rename Instrumentor/{qemu => }/build_qemu_x86.sh (87%) rename Instrumentor/{qemu => }/generate_bbcount_patch.sh (100%) rename Instrumentor/{qemu => }/generate_branch_patch.sh (100%) rename Instrumentor/{qemu => }/generate_common_patch.sh (100%) rename Instrumentor/{qemu => }/generate_coverage_patch.sh (100%) rename Instrumentor/{qemu => }/patches-bbcount/chatkey.cc (100%) rename Instrumentor/{qemu => }/patches-bbcount/cpu-exec.diff (100%) rename Instrumentor/{qemu => }/patches-bbcount/makefile-target.diff (100%) rename Instrumentor/{qemu => }/patches-bbcount/syscall.diff (100%) rename Instrumentor/{qemu => }/patches-branch/afl-qemu-cpu-inl.h (100%) rename Instrumentor/{qemu => }/patches-branch/chatkey.c (100%) rename Instrumentor/{qemu => }/patches-branch/cpu-exec.diff (100%) rename Instrumentor/{qemu => }/patches-branch/makefile-target.diff (100%) rename Instrumentor/{qemu => }/patches-branch/optimize.diff (100%) rename Instrumentor/{qemu => }/patches-branch/syscall.diff (100%) rename Instrumentor/{qemu => }/patches-branch/tcg-op.diff (100%) rename Instrumentor/{qemu => }/patches-branch/tcg-opc.diff (100%) rename Instrumentor/{qemu => }/patches-branch/tcg-target.diff (100%) rename Instrumentor/{qemu => }/patches-branch/tcg.diff (100%) rename Instrumentor/{qemu => }/patches-branch/translate.diff (100%) rename Instrumentor/{qemu => }/patches-common/configure.diff (100%) rename Instrumentor/{qemu => }/patches-common/elfload.diff (100%) rename Instrumentor/{qemu => }/patches-common/memfd.diff (100%) rename Instrumentor/{qemu => }/patches-common/signal.diff (100%) rename Instrumentor/{qemu => }/patches-coverage/afl-qemu-cpu-inl.h (100%) rename Instrumentor/{qemu => }/patches-coverage/chatkey.c (100%) rename Instrumentor/{qemu => }/patches-coverage/cpu-exec.diff (100%) rename Instrumentor/{qemu => }/patches-coverage/makefile-objs.diff (100%) rename Instrumentor/{qemu => }/patches-coverage/syscall.diff (100%) rename Instrumentor/{qemu => }/prepare_qemu.sh (100%) rename Instrumentor/{qemu => }/repatch.sh (100%) diff --git a/.gitignore b/.gitignore index c50317b..16bdef1 100644 --- a/.gitignore +++ b/.gitignore @@ -12,11 +12,11 @@ obj/ .vscode/ #etc -Instrumentor/qemu/.prepared -Instrumentor/qemu/qemu-2.10.0* -Instrumentor/qemu/.compiled_x86 -Instrumentor/qemu/.compiled_x64 -Instrumentor/qemu/.compiled +Instrumentor/.prepared +Instrumentor/qemu-2.10.0* +Instrumentor/.compiled_x86 +Instrumentor/.compiled_x64 +Instrumentor/.compiled *.swp *.bin box diff --git a/Instrumentor/qemu/build_qemu_x64.sh b/Instrumentor/build_qemu_x64.sh similarity index 87% rename from Instrumentor/qemu/build_qemu_x64.sh rename to Instrumentor/build_qemu_x64.sh index 73f9f61..7ac609a 100755 --- a/Instrumentor/qemu/build_qemu_x64.sh +++ b/Instrumentor/build_qemu_x64.sh @@ -41,15 +41,15 @@ build_qemu () { } build_qemu coverage -mv "./qemu-trace" "../../build/qemu-trace-coverage-x64" || exit 1 +mv "./qemu-trace" "../build/qemu-trace-coverage-x64" || exit 1 echo "[+] Successfully created 'qemu-trace-coverage-x64'." build_qemu branch -mv "./qemu-trace" "../../build/qemu-trace-branch-x64" || exit 1 +mv "./qemu-trace" "../build/qemu-trace-branch-x64" || exit 1 echo "[+] Successfully created 'qemu-trace-branch-x64'." build_qemu bbcount -mv "./qemu-trace" "../../build/qemu-trace-bbcount-x64" || exit 1 +mv "./qemu-trace" "../build/qemu-trace-bbcount-x64" || exit 1 echo "[+] Successfully created 'qemu-trace-bbcount-x64'." exit 0 diff --git a/Instrumentor/qemu/build_qemu_x86.sh b/Instrumentor/build_qemu_x86.sh similarity index 87% rename from Instrumentor/qemu/build_qemu_x86.sh rename to Instrumentor/build_qemu_x86.sh index 874714c..d37ab14 100755 --- a/Instrumentor/qemu/build_qemu_x86.sh +++ b/Instrumentor/build_qemu_x86.sh @@ -41,15 +41,15 @@ build_qemu () { } build_qemu coverage -mv "./qemu-trace" "../../build/qemu-trace-coverage-x86" || exit 1 +mv "./qemu-trace" "../build/qemu-trace-coverage-x86" || exit 1 echo "[+] Successfully created 'qemu-trace-coverage-x86'." build_qemu branch -mv "./qemu-trace" "../../build/qemu-trace-branch-x86" || exit 1 +mv "./qemu-trace" "../build/qemu-trace-branch-x86" || exit 1 echo "[+] Successfully created 'qemu-trace-branch-x86'." build_qemu bbcount -mv "./qemu-trace" "../../build/qemu-trace-bbcount-x86" || exit 1 +mv "./qemu-trace" "../build/qemu-trace-bbcount-x86" || exit 1 echo "[+] Successfully created 'qemu-trace-bbcount-x86'." exit 0 diff --git a/Instrumentor/qemu/generate_bbcount_patch.sh b/Instrumentor/generate_bbcount_patch.sh similarity index 100% rename from Instrumentor/qemu/generate_bbcount_patch.sh rename to Instrumentor/generate_bbcount_patch.sh diff --git a/Instrumentor/qemu/generate_branch_patch.sh b/Instrumentor/generate_branch_patch.sh similarity index 100% rename from Instrumentor/qemu/generate_branch_patch.sh rename to Instrumentor/generate_branch_patch.sh diff --git a/Instrumentor/qemu/generate_common_patch.sh b/Instrumentor/generate_common_patch.sh similarity index 100% rename from Instrumentor/qemu/generate_common_patch.sh rename to Instrumentor/generate_common_patch.sh diff --git a/Instrumentor/qemu/generate_coverage_patch.sh b/Instrumentor/generate_coverage_patch.sh similarity index 100% rename from Instrumentor/qemu/generate_coverage_patch.sh rename to Instrumentor/generate_coverage_patch.sh diff --git a/Instrumentor/qemu/patches-bbcount/chatkey.cc b/Instrumentor/patches-bbcount/chatkey.cc similarity index 100% rename from Instrumentor/qemu/patches-bbcount/chatkey.cc rename to Instrumentor/patches-bbcount/chatkey.cc diff --git a/Instrumentor/qemu/patches-bbcount/cpu-exec.diff b/Instrumentor/patches-bbcount/cpu-exec.diff similarity index 100% rename from Instrumentor/qemu/patches-bbcount/cpu-exec.diff rename to Instrumentor/patches-bbcount/cpu-exec.diff diff --git a/Instrumentor/qemu/patches-bbcount/makefile-target.diff b/Instrumentor/patches-bbcount/makefile-target.diff similarity index 100% rename from Instrumentor/qemu/patches-bbcount/makefile-target.diff rename to Instrumentor/patches-bbcount/makefile-target.diff diff --git a/Instrumentor/qemu/patches-bbcount/syscall.diff b/Instrumentor/patches-bbcount/syscall.diff similarity index 100% rename from Instrumentor/qemu/patches-bbcount/syscall.diff rename to Instrumentor/patches-bbcount/syscall.diff diff --git a/Instrumentor/qemu/patches-branch/afl-qemu-cpu-inl.h b/Instrumentor/patches-branch/afl-qemu-cpu-inl.h similarity index 100% rename from Instrumentor/qemu/patches-branch/afl-qemu-cpu-inl.h rename to Instrumentor/patches-branch/afl-qemu-cpu-inl.h diff --git a/Instrumentor/qemu/patches-branch/chatkey.c b/Instrumentor/patches-branch/chatkey.c similarity index 100% rename from Instrumentor/qemu/patches-branch/chatkey.c rename to Instrumentor/patches-branch/chatkey.c diff --git a/Instrumentor/qemu/patches-branch/cpu-exec.diff b/Instrumentor/patches-branch/cpu-exec.diff similarity index 100% rename from Instrumentor/qemu/patches-branch/cpu-exec.diff rename to Instrumentor/patches-branch/cpu-exec.diff diff --git a/Instrumentor/qemu/patches-branch/makefile-target.diff b/Instrumentor/patches-branch/makefile-target.diff similarity index 100% rename from Instrumentor/qemu/patches-branch/makefile-target.diff rename to Instrumentor/patches-branch/makefile-target.diff diff --git a/Instrumentor/qemu/patches-branch/optimize.diff b/Instrumentor/patches-branch/optimize.diff similarity index 100% rename from Instrumentor/qemu/patches-branch/optimize.diff rename to Instrumentor/patches-branch/optimize.diff diff --git a/Instrumentor/qemu/patches-branch/syscall.diff b/Instrumentor/patches-branch/syscall.diff similarity index 100% rename from Instrumentor/qemu/patches-branch/syscall.diff rename to Instrumentor/patches-branch/syscall.diff diff --git a/Instrumentor/qemu/patches-branch/tcg-op.diff b/Instrumentor/patches-branch/tcg-op.diff similarity index 100% rename from Instrumentor/qemu/patches-branch/tcg-op.diff rename to Instrumentor/patches-branch/tcg-op.diff diff --git a/Instrumentor/qemu/patches-branch/tcg-opc.diff b/Instrumentor/patches-branch/tcg-opc.diff similarity index 100% rename from Instrumentor/qemu/patches-branch/tcg-opc.diff rename to Instrumentor/patches-branch/tcg-opc.diff diff --git a/Instrumentor/qemu/patches-branch/tcg-target.diff b/Instrumentor/patches-branch/tcg-target.diff similarity index 100% rename from Instrumentor/qemu/patches-branch/tcg-target.diff rename to Instrumentor/patches-branch/tcg-target.diff diff --git a/Instrumentor/qemu/patches-branch/tcg.diff b/Instrumentor/patches-branch/tcg.diff similarity index 100% rename from Instrumentor/qemu/patches-branch/tcg.diff rename to Instrumentor/patches-branch/tcg.diff diff --git a/Instrumentor/qemu/patches-branch/translate.diff b/Instrumentor/patches-branch/translate.diff similarity index 100% rename from Instrumentor/qemu/patches-branch/translate.diff rename to Instrumentor/patches-branch/translate.diff diff --git a/Instrumentor/qemu/patches-common/configure.diff b/Instrumentor/patches-common/configure.diff similarity index 100% rename from Instrumentor/qemu/patches-common/configure.diff rename to Instrumentor/patches-common/configure.diff diff --git a/Instrumentor/qemu/patches-common/elfload.diff b/Instrumentor/patches-common/elfload.diff similarity index 100% rename from Instrumentor/qemu/patches-common/elfload.diff rename to Instrumentor/patches-common/elfload.diff diff --git a/Instrumentor/qemu/patches-common/memfd.diff b/Instrumentor/patches-common/memfd.diff similarity index 100% rename from Instrumentor/qemu/patches-common/memfd.diff rename to Instrumentor/patches-common/memfd.diff diff --git a/Instrumentor/qemu/patches-common/signal.diff b/Instrumentor/patches-common/signal.diff similarity index 100% rename from Instrumentor/qemu/patches-common/signal.diff rename to Instrumentor/patches-common/signal.diff diff --git a/Instrumentor/qemu/patches-coverage/afl-qemu-cpu-inl.h b/Instrumentor/patches-coverage/afl-qemu-cpu-inl.h similarity index 100% rename from Instrumentor/qemu/patches-coverage/afl-qemu-cpu-inl.h rename to Instrumentor/patches-coverage/afl-qemu-cpu-inl.h diff --git a/Instrumentor/qemu/patches-coverage/chatkey.c b/Instrumentor/patches-coverage/chatkey.c similarity index 100% rename from Instrumentor/qemu/patches-coverage/chatkey.c rename to Instrumentor/patches-coverage/chatkey.c diff --git a/Instrumentor/qemu/patches-coverage/cpu-exec.diff b/Instrumentor/patches-coverage/cpu-exec.diff similarity index 100% rename from Instrumentor/qemu/patches-coverage/cpu-exec.diff rename to Instrumentor/patches-coverage/cpu-exec.diff diff --git a/Instrumentor/qemu/patches-coverage/makefile-objs.diff b/Instrumentor/patches-coverage/makefile-objs.diff similarity index 100% rename from Instrumentor/qemu/patches-coverage/makefile-objs.diff rename to Instrumentor/patches-coverage/makefile-objs.diff diff --git a/Instrumentor/qemu/patches-coverage/syscall.diff b/Instrumentor/patches-coverage/syscall.diff similarity index 100% rename from Instrumentor/qemu/patches-coverage/syscall.diff rename to Instrumentor/patches-coverage/syscall.diff diff --git a/Instrumentor/qemu/prepare_qemu.sh b/Instrumentor/prepare_qemu.sh similarity index 100% rename from Instrumentor/qemu/prepare_qemu.sh rename to Instrumentor/prepare_qemu.sh diff --git a/Instrumentor/qemu/repatch.sh b/Instrumentor/repatch.sh similarity index 100% rename from Instrumentor/qemu/repatch.sh rename to Instrumentor/repatch.sh diff --git a/Makefile b/Makefile index b03ded5..ab2a20c 100644 --- a/Makefile +++ b/Makefile @@ -1,5 +1,6 @@ +QEMU_VERSION=2.10.0 BUILDDIR=$(shell pwd)/build -QEMUDIR=$(shell pwd)/Instrumentor/qemu +QEMUDIR=$(shell pwd)/Instrumentor all: $(BUILDDIR) $(QEMUDIR)/.compiled Eclipser @@ -12,8 +13,8 @@ clean: rm -f $(QEMUDIR)/.compiled rm -f $(QEMUDIR)/.compiled_x86 rm -f $(QEMUDIR)/.compiled_x64 - rm -rf $(QEMUDIR)/qemu-2.3.0 - rm -rf $(QEMUDIR)/qemu-2.3.0-* + rm -rf $(QEMUDIR)/qemu-${QEMU_VERSION} + rm -rf $(QEMUDIR)/qemu-${QEMU_VERSION}-* rm -rf $(BUILDDIR) $(BUILDDIR): From cea14123fa8f0c3f28fc8651a582f0e4a5d2e6d3 Mon Sep 17 00:00:00 2001 From: Jaeseung Choi Date: Mon, 5 Oct 2020 02:14:07 -0700 Subject: [PATCH 17/29] Cleanup instrumentation code Remove legacy name (Chatkey) from the code. --- Instrumentor/generate_bbcount_patch.sh | 2 +- Instrumentor/generate_branch_patch.sh | 2 +- Instrumentor/generate_coverage_patch.sh | 2 +- Instrumentor/patches-bbcount/cpu-exec.diff | 12 +- .../{chatkey.cc => eclipser.cc} | 31 ++-- .../patches-bbcount/makefile-target.diff | 2 +- Instrumentor/patches-bbcount/syscall.diff | 10 +- .../patches-branch/afl-qemu-cpu-inl.h | 10 +- Instrumentor/patches-branch/cpu-exec.diff | 16 +- .../patches-branch/{chatkey.c => eclipser.c} | 169 ++++++++++-------- .../patches-branch/makefile-target.diff | 2 +- Instrumentor/patches-branch/syscall.diff | 10 +- Instrumentor/patches-branch/tcg-op.diff | 58 +++--- Instrumentor/patches-branch/tcg-target.diff | 14 +- Instrumentor/patches-branch/tcg.diff | 30 +--- Instrumentor/patches-branch/translate.diff | 54 +++--- Instrumentor/patches-common/elfload.diff | 4 +- Instrumentor/patches-common/signal.diff | 6 +- .../patches-coverage/afl-qemu-cpu-inl.h | 2 +- Instrumentor/patches-coverage/cpu-exec.diff | 16 +- .../{chatkey.c => eclipser.c} | 35 ++-- .../patches-coverage/makefile-objs.diff | 2 +- Instrumentor/patches-coverage/syscall.diff | 10 +- Instrumentor/prepare_qemu.sh | 10 +- Instrumentor/repatch.sh | 12 +- src/Core/Executor.fs | 18 +- 26 files changed, 273 insertions(+), 266 deletions(-) rename Instrumentor/patches-bbcount/{chatkey.cc => eclipser.cc} (85%) rename Instrumentor/patches-branch/{chatkey.c => eclipser.c} (60%) rename Instrumentor/patches-coverage/{chatkey.c => eclipser.c} (81%) diff --git a/Instrumentor/generate_bbcount_patch.sh b/Instrumentor/generate_bbcount_patch.sh index a616a64..2671280 100755 --- a/Instrumentor/generate_bbcount_patch.sh +++ b/Instrumentor/generate_bbcount_patch.sh @@ -4,7 +4,7 @@ VERSION="2.10.0" cp -r qemu-${VERSION}-bbcount-x64 qemu-${VERSION}-bbcount -cp qemu-${VERSION}-bbcount/chatkey.cc ./patches-bbcount/ +cp qemu-${VERSION}-bbcount/eclipser.cc ./patches-bbcount/ cp qemu-${VERSION}/accel/tcg/cpu-exec.c \ qemu-${VERSION}-bbcount/accel/tcg/cpu-exec.c.orig diff --git a/Instrumentor/generate_branch_patch.sh b/Instrumentor/generate_branch_patch.sh index a6bd677..4fcc3e8 100755 --- a/Instrumentor/generate_branch_patch.sh +++ b/Instrumentor/generate_branch_patch.sh @@ -6,7 +6,7 @@ cp -r qemu-${VERSION}-branch-x64 qemu-${VERSION}-branch cp qemu-${VERSION}-branch/afl-qemu-cpu-inl.h ./patches-branch/afl-qemu-cpu-inl.h -cp qemu-${VERSION}-branch/tcg/chatkey.c ./patches-branch/chatkey.c +cp qemu-${VERSION}-branch/tcg/eclipser.c ./patches-branch/eclipser.c cp qemu-${VERSION}/accel/tcg/cpu-exec.c \ qemu-${VERSION}-branch/accel/tcg/cpu-exec.c.orig diff --git a/Instrumentor/generate_coverage_patch.sh b/Instrumentor/generate_coverage_patch.sh index a594e35..edec52f 100755 --- a/Instrumentor/generate_coverage_patch.sh +++ b/Instrumentor/generate_coverage_patch.sh @@ -6,7 +6,7 @@ cp -r qemu-${VERSION}-coverage-x64 qemu-${VERSION}-coverage cp qemu-${VERSION}-coverage/accel/tcg/afl-qemu-cpu-inl.h ./patches-coverage/ -cp qemu-${VERSION}-coverage/accel/tcg/chatkey.c ./patches-coverage/ +cp qemu-${VERSION}-coverage/accel/tcg/eclipser.c ./patches-coverage/ cp qemu-${VERSION}/accel/tcg/cpu-exec.c \ qemu-${VERSION}-coverage/accel/tcg/cpu-exec.c.orig diff --git a/Instrumentor/patches-bbcount/cpu-exec.diff b/Instrumentor/patches-bbcount/cpu-exec.diff index 405f109..1eea9b6 100644 --- a/Instrumentor/patches-bbcount/cpu-exec.diff +++ b/Instrumentor/patches-bbcount/cpu-exec.diff @@ -4,9 +4,9 @@ #include "sysemu/cpus.h" #include "sysemu/replay.h" -+extern abi_ulong chatkey_entry_point; -+extern void chatkey_setup(void); -+extern void chatkey_log_bb(abi_ulong addr); ++extern abi_ulong eclipser_entry_point; ++extern void eclipser_setup(void); ++extern void eclipser_log_bb(abi_ulong addr); + /* -icount align implementation. */ @@ -15,10 +15,10 @@ int tb_exit; uint8_t *tb_ptr = itb->tc_ptr; -+ if(itb->pc == chatkey_entry_point) { -+ chatkey_setup(); ++ if(itb->pc == eclipser_entry_point) { ++ eclipser_setup(); + } -+ chatkey_log_bb(itb->pc); ++ eclipser_log_bb(itb->pc); + qemu_log_mask_and_addr(CPU_LOG_EXEC, itb->pc, "Trace %p [%d: " TARGET_FMT_lx "] %s\n", diff --git a/Instrumentor/patches-bbcount/chatkey.cc b/Instrumentor/patches-bbcount/eclipser.cc similarity index 85% rename from Instrumentor/patches-bbcount/chatkey.cc rename to Instrumentor/patches-bbcount/eclipser.cc index 1bc5d0f..b0bb920 100644 --- a/Instrumentor/patches-bbcount/chatkey.cc +++ b/Instrumentor/patches-bbcount/eclipser.cc @@ -16,13 +16,13 @@ typedef uint64_t abi_ulong; static void init_accum_set(char* path, set *accum_set); -extern "C" void chatkey_setup(void); -extern "C" void chatkey_close_fp(void); -extern "C" void chatkey_exit(void); -extern "C" void chatkey_log_bb(abi_ulong node); +extern "C" void eclipser_setup(void); +extern "C" void eclipser_detach(void); +extern "C" void eclipser_exit(void); +extern "C" void eclipser_log_bb(abi_ulong node); /* ELF entry point (_start). */ -abi_ulong chatkey_entry_point; +abi_ulong eclipser_entry_point; static FILE* coverage_fp; static FILE* node_fp; @@ -61,15 +61,15 @@ static void init_accum_set(char* path, set *accum_set) { return; } -extern "C" void chatkey_setup(void) { +extern "C" void eclipser_setup(void) { /* Open file pointers and descriptors early, since if we try to open them in - * chatkey_exit(), it gets mixed with stderr & stdout stream. This seems to + * eclipser_exit(), it gets mixed with stderr & stdout stream. This seems to * be an issue due to incorrect file descriptor management in QEMU code. */ - char * coverage_log = getenv("CK_COVERAGE_LOG"); - char * node_log = getenv("CK_NODE_LOG"); - char * edge_log = getenv("CK_EDGE_LOG"); + char * coverage_log = getenv("ECL_COVERAGE_LOG"); + char * node_log = getenv("ECL_NODE_LOG"); + char * edge_log = getenv("ECL_EDGE_LOG"); assert(coverage_log != NULL); coverage_fp = fopen(coverage_log, "a"); @@ -97,7 +97,8 @@ extern "C" void chatkey_setup(void) { } // When fork() syscall is encountered, child process should call this function -extern "C" void chatkey_close_fp(void) { +// to detach from Eclipser. +extern "C" void eclipser_detach(void) { if (coverage_fp) { fclose(coverage_fp); coverage_fp = NULL; @@ -114,12 +115,12 @@ extern "C" void chatkey_close_fp(void) { } } -extern "C" void chatkey_exit(void) { +extern "C" void eclipser_exit(void) { sigset_t mask; unsigned int accum_node_cnt, accum_edge_cnt; unsigned int new_node_cnt, new_edge_cnt; - // Block signals, since we register signal handler that calls chatkey_exit()/ + // Block signals, since we register signal handler that calls eclipser_exit(). if (sigfillset(&mask) < 0) return; if (sigprocmask(SIG_BLOCK, &mask, NULL) < 0) @@ -151,10 +152,10 @@ extern "C" void chatkey_exit(void) { } } -extern "C" void chatkey_log_bb(abi_ulong node) { +extern "C" void eclipser_log_bb(abi_ulong node) { abi_ulong edge; - /* If coverage_fp is NULL, it means that chatkey_setup() is not called yet + /* If coverage_fp is NULL, it means that eclipser_setup() is not called yet * This happens when QEMU is executing a dynamically linked program. */ if (!coverage_fp || !node_fp || !edge_fp) diff --git a/Instrumentor/patches-bbcount/makefile-target.diff b/Instrumentor/patches-bbcount/makefile-target.diff index ad6c8a7..ed74e73 100644 --- a/Instrumentor/patches-bbcount/makefile-target.diff +++ b/Instrumentor/patches-bbcount/makefile-target.diff @@ -4,7 +4,7 @@ ######################################################### # cpu emulator library obj-y += exec.o -+obj-y += chatkey.o ++obj-y += eclipser.o obj-y += accel/ obj-$(CONFIG_TCG) += tcg/tcg.o tcg/tcg-op.o tcg/optimize.o obj-$(CONFIG_TCG) += tcg/tcg-common.o tcg/tcg-runtime.o diff --git a/Instrumentor/patches-bbcount/syscall.diff b/Instrumentor/patches-bbcount/syscall.diff index 34c9d3f..a04b16f 100644 --- a/Instrumentor/patches-bbcount/syscall.diff +++ b/Instrumentor/patches-bbcount/syscall.diff @@ -4,8 +4,8 @@ #include "qemu.h" -+extern void chatkey_exit(void); -+extern void chatkey_close_fp(void); ++extern void eclipser_exit(void); ++extern void eclipser_detach(void); + #ifndef CLONE_IO #define CLONE_IO 0x80000000 /* Clone io context */ @@ -14,7 +14,7 @@ ret = fork(); if (ret == 0) { /* Child Process. */ -+ chatkey_close_fp(); ++ eclipser_detach(); cpu_clone_regs(env, newsp); fork_end(1); /* There is a race condition here. The parent process could @@ -22,7 +22,7 @@ #ifdef TARGET_GPROF _mcleanup(); #endif -+ chatkey_exit(); ++ eclipser_exit(); gdb_exit(cpu_env, arg1); _exit(arg1); ret = 0; /* avoid warning */ @@ -30,7 +30,7 @@ #ifdef TARGET_GPROF _mcleanup(); #endif -+ chatkey_exit(); ++ eclipser_exit(); gdb_exit(cpu_env, arg1); ret = get_errno(exit_group(arg1)); break; diff --git a/Instrumentor/patches-branch/afl-qemu-cpu-inl.h b/Instrumentor/patches-branch/afl-qemu-cpu-inl.h index 3a3a44c..f9de5f2 100644 --- a/Instrumentor/patches-branch/afl-qemu-cpu-inl.h +++ b/Instrumentor/patches-branch/afl-qemu-cpu-inl.h @@ -23,8 +23,8 @@ #define FORKSRV_FD 194 #define TSL_FD (FORKSRV_FD - 1) -extern abi_ulong chatkey_targ_addr; -extern uint32_t chatkey_targ_index; +extern abi_ulong eclipser_targ_addr; +extern uint32_t eclipser_targ_index; extern int measure_coverage; /* Set in the child process in forkserver mode: */ @@ -58,7 +58,7 @@ static void afl_forkserver(CPUState *cpu) { static unsigned char tmp[4]; uint64_t tmp_input; - if (atoi(getenv("CK_FORK_SERVER")) != 1) return; + if (atoi(getenv("ECL_FORK_SERVER")) != 1) return; /* Tell the parent that we're alive. If the parent doesn't want to talk, assume that we're not running in forkserver mode. */ @@ -77,8 +77,8 @@ static void afl_forkserver(CPUState *cpu) { /* Whoops, parent dead? */ if (read(FORKSRV_FD, &tmp_input, 8) != 8) exit(2); - chatkey_targ_addr = (abi_ulong)tmp_input; - if (read(FORKSRV_FD, &chatkey_targ_index, 4) != 4) exit(2); + eclipser_targ_addr = (abi_ulong)tmp_input; + if (read(FORKSRV_FD, &eclipser_targ_index, 4) != 4) exit(2); if (read(FORKSRV_FD, &measure_coverage, 4) != 4) exit(2); /* Establish a channel with child to grab translation commands. We'll diff --git a/Instrumentor/patches-branch/cpu-exec.diff b/Instrumentor/patches-branch/cpu-exec.diff index fc87757..0d1d5b0 100644 --- a/Instrumentor/patches-branch/cpu-exec.diff +++ b/Instrumentor/patches-branch/cpu-exec.diff @@ -5,10 +5,10 @@ #include "sysemu/replay.h" +#include "afl-qemu-cpu-inl.h" -+extern abi_ulong chatkey_entry_point; /* ELF entry point (_start) */ -+extern void chatkey_setup_before_forkserver(void); -+extern void chatkey_setup_after_forkserver(void); -+extern void chatkey_log_bb(abi_ulong addr); ++extern abi_ulong eclipser_entry_point; /* ELF entry point (_start) */ ++extern void eclipser_setup_before_forkserver(void); ++extern void eclipser_setup_after_forkserver(void); ++extern void eclipser_log_bb(abi_ulong addr); + /* -icount align implementation. */ @@ -20,14 +20,14 @@ + abi_ulong entry_pc; + + entry_pc = itb->pc; -+ if(entry_pc == chatkey_entry_point) { -+ chatkey_setup_before_forkserver(); ++ if(entry_pc == eclipser_entry_point) { ++ eclipser_setup_before_forkserver(); + // Resolves util/rcu.c assertion error issue (cf. AFL-2.53b). + rcu_disable_atfork(); + afl_forkserver(cpu); -+ chatkey_setup_after_forkserver(); ++ eclipser_setup_after_forkserver(); + } -+ chatkey_log_bb(entry_pc); ++ eclipser_log_bb(entry_pc); qemu_log_mask_and_addr(CPU_LOG_EXEC, itb->pc, "Trace %p [%d: " TARGET_FMT_lx "] %s\n", diff --git a/Instrumentor/patches-branch/chatkey.c b/Instrumentor/patches-branch/eclipser.c similarity index 60% rename from Instrumentor/patches-branch/chatkey.c rename to Instrumentor/patches-branch/eclipser.c index 88713f6..d0f88b0 100644 --- a/Instrumentor/patches-branch/chatkey.c +++ b/Instrumentor/patches-branch/eclipser.c @@ -29,19 +29,19 @@ extern unsigned int afl_forksrv_pid; #define CUMULATIVE_COVERAGE 3 void flush_trace_buffer(void); -void chatkey_setup_before_forkserver(void); -void chatkey_setup_after_forkserver(void); -void chatkey_close_fp(void); -void chatkey_exit(void); -void chatkey_log_branch(abi_ulong oprnd1, abi_ulong oprnd2, unsigned char type); -void chatkey_log_bb(abi_ulong addr); - -abi_ulong chatkey_entry_point = 0; /* ELF entry point (_start) */ -abi_ulong chatkey_curr_addr = 0; -abi_ulong chatkey_targ_addr = 0; -uint32_t chatkey_targ_index = 0; +void eclipser_setup_before_forkserver(void); +void eclipser_setup_after_forkserver(void); +void eclipser_detach(void); +void eclipser_exit(void); +void eclipser_log_branch(abi_ulong oprnd1, abi_ulong oprnd2, unsigned char type); +void eclipser_log_bb(abi_ulong addr); + +abi_ulong eclipser_entry_point = 0; /* ELF entry point (_start) */ +abi_ulong eclipser_curr_addr = 0; +abi_ulong eclipser_targ_addr = 0; +uint32_t eclipser_targ_index = 0; int measure_coverage = 0; -int chatkey_EP_passed = 0; +int eclipser_EP_passed = 0; static int found_new_edge = 0; static int found_new_path = 0; // TODO. Extend to measure path coverage, too. @@ -63,26 +63,26 @@ void flush_trace_buffer(void) { fwrite(trace_buffer, len, 1, branch_fp); } -void chatkey_setup_before_forkserver(void) { - char * bitmap_path = getenv("CK_BITMAP_LOG"); +void eclipser_setup_before_forkserver(void) { + char * bitmap_path = getenv("ECL_BITMAP_LOG"); int bitmap_fd = open(bitmap_path, O_RDWR | O_CREAT, 0644); edge_bitmap = (unsigned char*) mmap(NULL, 0x10000, PROT_READ | PROT_WRITE, MAP_SHARED, bitmap_fd, 0); assert(edge_bitmap != (void *) -1); - coverage_path = getenv("CK_COVERAGE_LOG"); - branch_path = getenv("CK_FEED_LOG"); + coverage_path = getenv("ECL_COVERAGE_LOG"); + branch_path = getenv("ECL_BRANCH_LOG"); - chatkey_EP_passed = 1; + eclipser_EP_passed = 1; } -void chatkey_setup_after_forkserver(void) { +void eclipser_setup_after_forkserver(void) { - assert(getenv("CK_FORK_SERVER") != NULL); + assert(getenv("ECL_FORK_SERVER") != NULL); // If fork server is enabled, the following data are set during the handshake. - if (atoi(getenv("CK_FORK_SERVER")) == 0) { - chatkey_targ_addr = strtol(getenv("CK_FEED_ADDR"), NULL, 16); - chatkey_targ_index = strtol(getenv("CK_FEED_IDX"), NULL, 16); - measure_coverage = atoi(getenv("CK_MEASURE_COV")); + if (atoi(getenv("ECL_FORK_SERVER")) == 0) { + eclipser_targ_addr = strtol(getenv("ECL_BRANCH_ADDR"), NULL, 16); + eclipser_targ_index = strtol(getenv("ECL_BRANCH_IDX"), NULL, 16); + measure_coverage = atoi(getenv("ECL_MEASURE_COV")); } if (measure_coverage != IGNORE_COVERAGE) { @@ -94,8 +94,9 @@ void chatkey_setup_after_forkserver(void) { assert(branch_fp != NULL); } -// When fork() syscall is encountered, child process should call this function. -void chatkey_close_fp(void) { +// When fork() syscall is encountered, child process should call this function +// to detach from Eclipser. +void eclipser_detach(void) { // Close file pointers, to avoid dumping log twice. if (coverage_fp) { fclose(coverage_fp); @@ -116,11 +117,11 @@ void chatkey_close_fp(void) { } } -void chatkey_exit(void) { +void eclipser_exit(void) { abi_ulong nil = 0; sigset_t mask; - // Block signals, since we register signal handler that calls chatkey_exit() + // Block signals, since we register signal handler that calls eclipser_exit() if (sigfillset(&mask) < 0) return; if (sigprocmask(SIG_BLOCK, &mask, NULL) < 0) @@ -146,30 +147,30 @@ void chatkey_exit(void) { } /* Recall that in 64bit we already pushed rdi/rsi/rdx before calling - * chatkey_trampline(). + * eclipser_trampline(). */ -asm (".global chatkey_trampoline \t\n\ - .type chatkey_trampoline, @function \t\n\ - chatkey_trampoline: \t\n\ - push %rax \t\n\ - push %rcx \t\n\ - push %r8 \t\n\ - push %r9 \t\n\ - push %r10 \t\n\ - push %r11 \t\n\ - call chatkey_log_branch; \t\n\ - pop %r11 \t\n\ - pop %r10 \t\n\ - pop %r9 \t\n\ - pop %r8 \t\n\ - pop %rcx \t\n\ - pop %rax \t\n\ - ret \t\n\ - .size chatkey_trampoline, . - chatkey_trampoline \t\n\ +asm (".global eclipser_trampoline \t\n\ + .type eclipser_trampoline, @function \t\n\ + eclipser_trampoline: \t\n\ + push %rax \t\n\ + push %rcx \t\n\ + push %r8 \t\n\ + push %r9 \t\n\ + push %r10 \t\n\ + push %r11 \t\n\ + call eclipser_log_branch; \t\n\ + pop %r11 \t\n\ + pop %r10 \t\n\ + pop %r9 \t\n\ + pop %r8 \t\n\ + pop %rcx \t\n\ + pop %rax \t\n\ + ret \t\n\ + .size eclipser_trampoline, . - eclipser_trampoline \t\n\ "); -void chatkey_log_branch(abi_ulong oprnd1, abi_ulong oprnd2, unsigned char type) { - +void eclipser_log_branch(abi_ulong oprnd1, abi_ulong oprnd2, unsigned char type) +{ abi_ulong oprnd1_truncated, oprnd2_truncated; unsigned char operand_type = type & 0x3f; unsigned char compare_type = type & 0xc0; @@ -178,10 +179,10 @@ void chatkey_log_branch(abi_ulong oprnd1, abi_ulong oprnd2, unsigned char type) if (!branch_fp) return; - if (chatkey_targ_addr) { + if (eclipser_targ_addr) { /* We're in the mode that traces cmp/test at a specific address */ - if (chatkey_curr_addr == chatkey_targ_addr && - ++targ_hit_count == chatkey_targ_index) { // Note that index starts from 1. + if (eclipser_curr_addr == eclipser_targ_addr && + ++targ_hit_count == eclipser_targ_index) { // Index starts from 1. if (operand_type == MO_8) { oprnd1_truncated = oprnd1 & 0xff; oprnd2_truncated = oprnd2 & 0xff; @@ -213,7 +214,7 @@ void chatkey_log_branch(abi_ulong oprnd1, abi_ulong oprnd2, unsigned char type) } type = compare_type | operand_size; - fwrite(&chatkey_curr_addr, sizeof(abi_ulong), 1, branch_fp); + fwrite(&eclipser_curr_addr, sizeof(abi_ulong), 1, branch_fp); fwrite(&type, sizeof(unsigned char), 1, branch_fp); fwrite(&oprnd1_truncated, operand_size, 1, branch_fp); fwrite(&oprnd2_truncated, operand_size, 1, branch_fp); @@ -223,75 +224,97 @@ void chatkey_log_branch(abi_ulong oprnd1, abi_ulong oprnd2, unsigned char type) * we are interested in branch distance only, and not in exit signal or * coverage gain. In these case, halt the execution here to save time. */ - chatkey_exit(); + eclipser_exit(); exit(0); } } } else if (trace_count++ < MAX_TRACE_LEN) { /* We're in the mode that traces all the cmp/test instructions */ + // First log the current address. + * (abi_ulong*) buf_ptr = eclipser_curr_addr; + buf_ptr += sizeof(abi_ulong); if (operand_type == MO_8) { oprnd1_truncated = oprnd1 & 0xff; oprnd2_truncated = oprnd2 & 0xff; operand_size = 1; - * (buf_ptr + sizeof(abi_ulong) + sizeof(unsigned char)) = oprnd1_truncated; - * (buf_ptr + sizeof(abi_ulong) + sizeof(unsigned char) + operand_size) = oprnd2_truncated; + type = compare_type | operand_size; + *buf_ptr = type; + buf_ptr += sizeof(unsigned char); + *buf_ptr = oprnd1_truncated; + buf_ptr += operand_size; + *buf_ptr = oprnd2_truncated; + buf_ptr += operand_size; } else if (operand_type == MO_16) { oprnd1_truncated = oprnd1 & 0xffff; oprnd2_truncated = oprnd2 & 0xffff; operand_size = 2; - * (unsigned short*) (buf_ptr + sizeof(abi_ulong) + sizeof(unsigned char)) = oprnd1_truncated; - * (unsigned short*) (buf_ptr + sizeof(abi_ulong) + sizeof(unsigned char) + operand_size) = oprnd2_truncated; + type = compare_type | operand_size; + *buf_ptr = type; + buf_ptr += sizeof(unsigned char); + * (unsigned short *) (buf_ptr) = oprnd1_truncated; + buf_ptr += operand_size; + * (unsigned short *) (buf_ptr) = oprnd2_truncated; + buf_ptr += operand_size; } #ifdef TARGET_X86_64 else if (operand_type == MO_32) { oprnd1_truncated = oprnd1 & 0xffffffff; oprnd2_truncated = oprnd2 & 0xffffffff; operand_size = 4; - * (unsigned int*) (buf_ptr + sizeof(abi_ulong) + sizeof(unsigned char)) = oprnd1_truncated; - * (unsigned int*) (buf_ptr + sizeof(abi_ulong) + sizeof(unsigned char) + operand_size) = oprnd2_truncated; + type = compare_type | operand_size; + *buf_ptr = type; + buf_ptr += sizeof(unsigned char); + * (unsigned int *) (buf_ptr) = oprnd1_truncated; + buf_ptr += operand_size; + * (unsigned int *) (buf_ptr) = oprnd2_truncated; + buf_ptr += operand_size; } else if (operand_type == MO_64) { oprnd1_truncated = oprnd1; oprnd2_truncated = oprnd2; operand_size = 8; - * (uint64_t*) (buf_ptr + sizeof(abi_ulong) + sizeof(unsigned char)) = oprnd1_truncated; - * (uint64_t*) (buf_ptr + sizeof(abi_ulong) + sizeof(unsigned char) + operand_size) = oprnd2_truncated; + type = compare_type | operand_size; + *buf_ptr = type; + buf_ptr += sizeof(unsigned char); + * (uint64_t *) (buf_ptr) = oprnd1_truncated; + buf_ptr += operand_size; + * (uint64_t *) (buf_ptr) = oprnd2_truncated; + buf_ptr += operand_size; } #else else if (operand_type == MO_32) { oprnd1_truncated = oprnd1; oprnd2_truncated = oprnd2; operand_size = 4; - * (unsigned int*) (buf_ptr + sizeof(abi_ulong) + sizeof(unsigned char)) = oprnd1_truncated; - * (unsigned int*) (buf_ptr + sizeof(abi_ulong) + sizeof(unsigned char) + operand_size) = oprnd2_truncated; + type = compare_type | operand_size; + *buf_ptr = type; + buf_ptr += sizeof(unsigned char); + * (unsigned int*) (buf_ptr) = oprnd1_truncated; + buf_ptr += operand_size; + * (unsigned int*) (buf_ptr) = oprnd2_truncated; + buf_ptr += operand_size; } #endif else { assert(0); } - type = compare_type | operand_size; - - * (abi_ulong*) buf_ptr = chatkey_curr_addr; - * (buf_ptr + sizeof(abi_ulong)) = type; - buf_ptr += sizeof(abi_ulong) + sizeof(unsigned char) + 2 * operand_size; - } else { /* We're in the mode that traces all the cmp/test instructions, and trace * limit has exceeded. Abort tracing. */ - chatkey_exit(); + eclipser_exit(); exit(0); } } -void chatkey_log_bb(abi_ulong addr) { +void eclipser_log_bb(abi_ulong addr) { abi_ulong prev_addr_local; abi_ulong edge, hash; unsigned int byte_idx, byte_mask; unsigned char old_byte, new_byte; - // Make sure that 'chatkey_curr_addr' and 'prev_addr' are always updated even + // Make sure that 'eclipser_curr_addr' and 'prev_addr' are always updated even // if we just return. - chatkey_curr_addr = addr; + eclipser_curr_addr = addr; prev_addr_local = prev_addr; prev_addr = addr; diff --git a/Instrumentor/patches-branch/makefile-target.diff b/Instrumentor/patches-branch/makefile-target.diff index b44f85f..8ccea59 100644 --- a/Instrumentor/patches-branch/makefile-target.diff +++ b/Instrumentor/patches-branch/makefile-target.diff @@ -5,7 +5,7 @@ obj-y += accel/ obj-$(CONFIG_TCG) += tcg/tcg.o tcg/tcg-op.o tcg/optimize.o -obj-$(CONFIG_TCG) += tcg/tcg-common.o tcg/tcg-runtime.o -+obj-$(CONFIG_TCG) += tcg/tcg-common.o tcg/tcg-runtime.o tcg/chatkey.o ++obj-$(CONFIG_TCG) += tcg/tcg-common.o tcg/tcg-runtime.o tcg/eclipser.o obj-$(CONFIG_TCG_INTERPRETER) += tcg/tci.o obj-$(CONFIG_TCG_INTERPRETER) += disas/tci.o obj-y += fpu/softfloat.o diff --git a/Instrumentor/patches-branch/syscall.diff b/Instrumentor/patches-branch/syscall.diff index 00b66aa..cb34aad 100644 --- a/Instrumentor/patches-branch/syscall.diff +++ b/Instrumentor/patches-branch/syscall.diff @@ -4,8 +4,8 @@ #include "qemu.h" -+extern void chatkey_exit(void); -+extern void chatkey_close_fp(void); ++extern void eclipser_exit(void); ++extern void eclipser_detach(void); +extern unsigned int afl_forksrv_pid; + #ifndef CLONE_IO @@ -15,7 +15,7 @@ ret = fork(); if (ret == 0) { /* Child Process. */ -+ chatkey_close_fp(); ++ eclipser_detach(); cpu_clone_regs(env, newsp); fork_end(1); /* There is a race condition here. The parent process could @@ -23,7 +23,7 @@ #ifdef TARGET_GPROF _mcleanup(); #endif -+ chatkey_exit(); ++ eclipser_exit(); gdb_exit(cpu_env, arg1); _exit(arg1); ret = 0; /* avoid warning */ @@ -31,7 +31,7 @@ #ifdef TARGET_GPROF _mcleanup(); #endif -+ chatkey_exit(); ++ eclipser_exit(); gdb_exit(cpu_env, arg1); ret = get_errno(exit_group(arg1)); break; diff --git a/Instrumentor/patches-branch/tcg-op.diff b/Instrumentor/patches-branch/tcg-op.diff index dbfb0c3..e001c84 100644 --- a/Instrumentor/patches-branch/tcg-op.diff +++ b/Instrumentor/patches-branch/tcg-op.diff @@ -4,10 +4,10 @@ #include "exec/helper-proto.h" #include "exec/helper-gen.h" -+#define CHATKEY_CMP_EQUALITY 0 -+#define CHATKEY_CMP_SIZE_SIGNED 1 -+#define CHATKEY_CMP_SIZE_UNSIGNED 2 -+#define CHATKEY_IGNORE 3 ++#define ECLIPSER_CMP_EQUALITY 0 ++#define ECLIPSER_CMP_SIZE_SIGNED 1 ++#define ECLIPSER_CMP_SIZE_UNSIGNED 2 ++#define ECLIPSER_IGNORE 3 + /* Basic output routines. Not for general consumption. */ @@ -16,38 +16,38 @@ tcg_gen_op3_i32(INDEX_op_add_i32, ret, arg1, arg2); } -+/* A variant of tcg_gen_sub_i32() for Chatkey. Additional arguments are passed ++/* A variant of tcg_gen_sub_i32() for Eclipser. Additional arguments are passed + * to tcg_gen_op5ii_i32(), to indicate the type and size of comparison. We first -+ * pass CHATKEY_IGNORE as comparison type, and update it later. ++ * pass ECLIPSER_IGNORE as comparison type, and update it later. + */ -+static inline void tcg_gen_sub_i32_chatkey(TCGv_i32 ret, TCGv_i32 arg1, ++static inline void tcg_gen_sub_i32_eclipser(TCGv_i32 ret, TCGv_i32 arg1, + TCGv_i32 arg2, TCGMemOp ot) +{ -+ tcg_gen_op5ii_i32(INDEX_op_sub_i32, ret, arg1, arg2, CHATKEY_IGNORE, ot); ++ tcg_gen_op5ii_i32(INDEX_op_sub_i32, ret, arg1, arg2, ECLIPSER_IGNORE, ot); +} + static inline void tcg_gen_sub_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2) { - tcg_gen_op3_i32(INDEX_op_sub_i32, ret, arg1, arg2); + // Fixed for compatibility. Use dummay values as arguments. -+ tcg_gen_op5ii_i32(INDEX_op_sub_i32, ret, arg1, arg2, CHATKEY_IGNORE, MO_8); ++ tcg_gen_op5ii_i32(INDEX_op_sub_i32, ret, arg1, arg2, ECLIPSER_IGNORE, MO_8); +} + -+/* A variant of tcg_gen_and_i32() for Chatkey. Additional arguments are passed ++/* A variant of tcg_gen_and_i32() for Eclipser. Additional arguments are passed + * to tcg_gen_op5ii_i32(), to indicate the type and size of comparison. We first -+ * pass CHATKEY_IGNORE as comparison type, and update it later. ++ * pass ECLIPSER_IGNORE as comparison type, and update it later. + */ -+static inline void tcg_gen_and_i32_chatkey(TCGv_i32 ret, TCGv_i32 arg1, ++static inline void tcg_gen_and_i32_eclipser(TCGv_i32 ret, TCGv_i32 arg1, + TCGv_i32 arg2, TCGMemOp ot) +{ -+ tcg_gen_op5ii_i32(INDEX_op_and_i32, ret, arg1, arg2, CHATKEY_IGNORE, ot); ++ tcg_gen_op5ii_i32(INDEX_op_and_i32, ret, arg1, arg2, ECLIPSER_IGNORE, ot); } static inline void tcg_gen_and_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2) { - tcg_gen_op3_i32(INDEX_op_and_i32, ret, arg1, arg2); + // Fixed for compatibility. Use dummay values as arguments. -+ tcg_gen_op5ii_i32(INDEX_op_and_i32, ret, arg1, arg2, CHATKEY_IGNORE, MO_8); ++ tcg_gen_op5ii_i32(INDEX_op_and_i32, ret, arg1, arg2, ECLIPSER_IGNORE, MO_8); } static inline void tcg_gen_or_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2) @@ -55,38 +55,38 @@ tcg_gen_op3_i64(INDEX_op_add_i64, ret, arg1, arg2); } -+/* A variant of tcg_gen_sub_i64() for Chatkey. Additional arguments are passed ++/* A variant of tcg_gen_sub_i64() for Eclipser. Additional arguments are passed + * to tcg_gen_op5ii_i64(), to indicate the type and size of comparison. We first -+ * pass CHATKEY_IGNORE as comparison type, and update it later. ++ * pass ECLIPSER_IGNORE as comparison type, and update it later. + */ -+static inline void tcg_gen_sub_i64_chatkey(TCGv_i64 ret, TCGv_i64 arg1, ++static inline void tcg_gen_sub_i64_eclipser(TCGv_i64 ret, TCGv_i64 arg1, + TCGv_i64 arg2, TCGMemOp ot) +{ -+ tcg_gen_op5ii_i64(INDEX_op_sub_i64, ret, arg1, arg2, CHATKEY_IGNORE, ot); ++ tcg_gen_op5ii_i64(INDEX_op_sub_i64, ret, arg1, arg2, ECLIPSER_IGNORE, ot); +} + static inline void tcg_gen_sub_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2) { - tcg_gen_op3_i64(INDEX_op_sub_i64, ret, arg1, arg2); + // Fixed for compatibility. Use dummay values as arguments. -+ tcg_gen_op5ii_i64(INDEX_op_sub_i64, ret, arg1, arg2, CHATKEY_IGNORE, MO_8); ++ tcg_gen_op5ii_i64(INDEX_op_sub_i64, ret, arg1, arg2, ECLIPSER_IGNORE, MO_8); +} + -+/* A variant of tcg_gen_and_i64() for Chatkey. Additional arguments are passed ++/* A variant of tcg_gen_and_i64() for Eclipser. Additional arguments are passed + * to tcg_gen_op5ii_i64(), to indicate the type and size of comparison. We first -+ * pass CHATKEY_IGNORE as comparison type, and update it later. ++ * pass ECLIPSER_IGNORE as comparison type, and update it later. + */ -+static inline void tcg_gen_and_i64_chatkey(TCGv_i64 ret, TCGv_i64 arg1, ++static inline void tcg_gen_and_i64_eclipser(TCGv_i64 ret, TCGv_i64 arg1, + TCGv_i64 arg2, TCGMemOp ot) +{ -+ tcg_gen_op5ii_i64(INDEX_op_and_i64, ret, arg1, arg2, CHATKEY_IGNORE, ot); ++ tcg_gen_op5ii_i64(INDEX_op_and_i64, ret, arg1, arg2, ECLIPSER_IGNORE, ot); } static inline void tcg_gen_and_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2) { - tcg_gen_op3_i64(INDEX_op_and_i64, ret, arg1, arg2); + // Fixed for compatibility. Use dummay values as arguments. -+ tcg_gen_op5ii_i64(INDEX_op_and_i64, ret, arg1, arg2, CHATKEY_IGNORE, MO_8); ++ tcg_gen_op5ii_i64(INDEX_op_and_i64, ret, arg1, arg2, ECLIPSER_IGNORE, MO_8); } static inline void tcg_gen_or_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2) @@ -98,7 +98,7 @@ + * architecture. This definition is used when both 'TCG_TARGET_REG_BITS == 32' + * and 'TARGET_LONG_BITS == 64' are satisfied, but this is infeasible according + * to target/i386/cpu.h and tcg/i386/tcg-target.h files. If we want to extend -+ * Chatkey to support other architectures, we should investigate if the ++ * eclipser to support other architectures, we should investigate if the + * architecture has 32-bit width registers but supports 64-bit subtraction. If + * so, below code must be modified as well. + */ @@ -112,12 +112,12 @@ #define tcg_gen_add_tl tcg_gen_add_i64 #define tcg_gen_addi_tl tcg_gen_addi_i64 #define tcg_gen_sub_tl tcg_gen_sub_i64 -+#define tcg_gen_sub_tl_chatkey tcg_gen_sub_i64_chatkey ++#define tcg_gen_sub_tl_eclipser tcg_gen_sub_i64_eclipser #define tcg_gen_neg_tl tcg_gen_neg_i64 #define tcg_gen_subfi_tl tcg_gen_subfi_i64 #define tcg_gen_subi_tl tcg_gen_subi_i64 #define tcg_gen_and_tl tcg_gen_and_i64 -+#define tcg_gen_and_tl_chatkey tcg_gen_and_i64_chatkey ++#define tcg_gen_and_tl_eclipser tcg_gen_and_i64_eclipser #define tcg_gen_andi_tl tcg_gen_andi_i64 #define tcg_gen_or_tl tcg_gen_or_i64 #define tcg_gen_ori_tl tcg_gen_ori_i64 @@ -125,12 +125,12 @@ #define tcg_gen_add_tl tcg_gen_add_i32 #define tcg_gen_addi_tl tcg_gen_addi_i32 #define tcg_gen_sub_tl tcg_gen_sub_i32 -+#define tcg_gen_sub_tl_chatkey tcg_gen_sub_i32_chatkey ++#define tcg_gen_sub_tl_eclipser tcg_gen_sub_i32_eclipser #define tcg_gen_neg_tl tcg_gen_neg_i32 #define tcg_gen_subfi_tl tcg_gen_subfi_i32 #define tcg_gen_subi_tl tcg_gen_subi_i32 #define tcg_gen_and_tl tcg_gen_and_i32 -+#define tcg_gen_and_tl_chatkey tcg_gen_and_i32_chatkey ++#define tcg_gen_and_tl_eclipser tcg_gen_and_i32_eclipser #define tcg_gen_andi_tl tcg_gen_andi_i32 #define tcg_gen_or_tl tcg_gen_or_i32 #define tcg_gen_ori_tl tcg_gen_ori_i32 diff --git a/Instrumentor/patches-branch/tcg-target.diff b/Instrumentor/patches-branch/tcg-target.diff index 82a054e..af9de98 100644 --- a/Instrumentor/patches-branch/tcg-target.diff +++ b/Instrumentor/patches-branch/tcg-target.diff @@ -4,7 +4,7 @@ #include "tcg-be-ldst.h" -+extern void chatkey_trampoline(abi_ulong oprnd1, ++extern void eclipser_trampoline(abi_ulong oprnd1, + abi_ulong oprnd2, + unsigned char type); + @@ -23,7 +23,7 @@ c = ARITH_ADD; goto gen_arith; OP_32_64(sub): -+ if (args[3] != CHATKEY_IGNORE) { ++ if (args[3] != ECLIPSER_IGNORE) { + ot = args[4]; +#if TCG_TARGET_REG_BITS == 64 + tcg_out_push(s, TCG_REG_RDI); @@ -36,7 +36,7 @@ + tcg_out_mov(s, TCG_TYPE_I64, TCG_REG_RSI, args[2]); + } + tcg_out_movi(s, TCG_TYPE_I64, TCG_REG_RDX, (args[3] << 6) | ot); -+ tcg_out_call(s, (tcg_insn_unit*)chatkey_trampoline); ++ tcg_out_call(s, (tcg_insn_unit*)eclipser_trampoline); + tcg_out_pop(s, TCG_REG_RDX); + tcg_out_pop(s, TCG_REG_RSI); + tcg_out_pop(s, TCG_REG_RDI); @@ -51,7 +51,7 @@ + tcg_out_mov(s, TCG_TYPE_I32, TCG_REG_ESI, args[2]); + } + tcg_out_movi(s, TCG_TYPE_I32, TCG_REG_EDX, (args[3] << 6) | ot); -+ tcg_out_call(s, (tcg_insn_unit*)chatkey_trampoline); ++ tcg_out_call(s, (tcg_insn_unit*)eclipser_trampoline); + tcg_out_pop(s, TCG_REG_EDX); + tcg_out_pop(s, TCG_REG_ESI); + tcg_out_pop(s, TCG_REG_EDI); @@ -63,7 +63,7 @@ + /* Recall that we only marked 'test r1, r1' case as our instrumentation + * target, so consider it as 'cmp r1, 0'. + */ -+ if (args[3] != CHATKEY_IGNORE) { ++ if (args[3] != ECLIPSER_IGNORE) { + ot = args[4]; +#if TCG_TARGET_REG_BITS == 64 + tcg_out_push(s, TCG_REG_RDI); @@ -72,7 +72,7 @@ + tcg_out_mov(s, TCG_TYPE_I64, TCG_REG_RDI, args[0]); + tcg_out_movi(s, TCG_TYPE_I64, TCG_REG_RSI, 0); + tcg_out_movi(s, TCG_TYPE_I64, TCG_REG_RDX, (args[3] << 6) | ot); -+ tcg_out_call(s, (tcg_insn_unit*)chatkey_trampoline); ++ tcg_out_call(s, (tcg_insn_unit*)eclipser_trampoline); + tcg_out_pop(s, TCG_REG_RDX); + tcg_out_pop(s, TCG_REG_RSI); + tcg_out_pop(s, TCG_REG_RDI); @@ -83,7 +83,7 @@ + tcg_out_mov(s, TCG_TYPE_I32, TCG_REG_EDI, args[0]); + tcg_out_movi(s, TCG_TYPE_I32, TCG_REG_ESI, 0); + tcg_out_movi(s, TCG_TYPE_I32, TCG_REG_EDX, (args[3] << 6) | ot); -+ tcg_out_call(s, (tcg_insn_unit*)chatkey_trampoline); ++ tcg_out_call(s, (tcg_insn_unit*)eclipser_trampoline); + tcg_out_pop(s, TCG_REG_EDX); + tcg_out_pop(s, TCG_REG_ESI); + tcg_out_pop(s, TCG_REG_EDI); diff --git a/Instrumentor/patches-branch/tcg.diff b/Instrumentor/patches-branch/tcg.diff index fe3dc18..5f81399 100644 --- a/Instrumentor/patches-branch/tcg.diff +++ b/Instrumentor/patches-branch/tcg.diff @@ -4,33 +4,15 @@ #include "qemu/bitops.h" #include "tcg-target.h" -+/* Constants used for Chatkey instrumentation. Specifies whether an instruction -+ * is target of Chatkey instrumentation (i.e. decides whether to insert code ++/* Constants used for Eclipser instrumentation. Specifies whether an instruction ++ * is target of Eclipser instrumentation (i.e. decides whether to insert code + * that records Eflags register value). + */ -+#define CHATKEY_CMP_EQUALITY 0 -+#define CHATKEY_CMP_SIZE_SIGNED 1 -+#define CHATKEY_CMP_SIZE_UNSIGNED 2 -+#define CHATKEY_IGNORE 3 ++#define ECLIPSER_CMP_EQUALITY 0 ++#define ECLIPSER_CMP_SIZE_SIGNED 1 ++#define ECLIPSER_CMP_SIZE_UNSIGNED 2 ++#define ECLIPSER_IGNORE 3 + /* Default target word size to pointer size. */ #ifndef TCG_TARGET_REG_BITS # if UINTPTR_MAX == UINT32_MAX -@@ -164,7 +173,7 @@ - int type; - tcg_insn_unit *ptr; - intptr_t addend; --} TCGRelocation; -+} TCGRelocation; - - typedef struct TCGLabel { - unsigned has_value : 1; -@@ -485,7 +494,7 @@ - uint8_t *op_sync_args; /* for each operation, each bit tells if the - corresponding output argument needs to be - sync to memory. */ -- -+ - TCGRegSet reserved_regs; - intptr_t current_frame_offset; - intptr_t frame_start; diff --git a/Instrumentor/patches-branch/translate.diff b/Instrumentor/patches-branch/translate.diff index 8854916..83bac2e 100644 --- a/Instrumentor/patches-branch/translate.diff +++ b/Instrumentor/patches-branch/translate.diff @@ -4,9 +4,9 @@ //#define MACRO_TEST 1 -+extern int chatkey_EP_passed; -+extern abi_ulong chatkey_curr_addr; -+extern abi_ulong chatkey_targ_addr; ++extern int eclipser_EP_passed; ++extern abi_ulong eclipser_curr_addr; ++extern abi_ulong eclipser_targ_addr; + /* global register indexes */ static TCGv_env cpu_env; @@ -15,8 +15,8 @@ int cpuid_ext3_features; int cpuid_7_0_ebx_features; int cpuid_xsave_features; -+ /* Additional field for Chatkey, to record the parameter index of the latest -+ * cmp/test instruction. ++ /* Additional field for eclipser, to record the parameter index of the ++ * latest cmp/test instruction. + */ + int latest_tgt_parm_idx; } DisasContext; @@ -26,9 +26,9 @@ tcg_gen_mov_tl(cpu_cc_dst, cpu_T0); } -+static inline void gen_op_testl_T0_T1_cc_chatkey(TCGMemOp ot) ++static inline void gen_op_testl_T0_T1_cc_eclipser(TCGMemOp ot) +{ -+ tcg_gen_and_tl_chatkey(cpu_cc_dst, cpu_T0, cpu_T1, ot); ++ tcg_gen_and_tl_eclipser(cpu_cc_dst, cpu_T0, cpu_T1, ot); +} + static inline void gen_op_testl_T0_T1_cc(void) @@ -92,17 +92,17 @@ + int cc_optimize_flag = 1; + + /* If this is a JCC instruction followed by sub/cmp/test, replace the -+ * previous comparison type 'CHATKEY_IGNORE' with a new one, to indicate ++ * previous comparison type 'ECLIPSER_IGNORE' with a new one, to indicate + * that this is our instrumentation target. + */ + if (jcc_op == JCC_Z && s->latest_tgt_parm_idx != -1) { -+ tcg_ctx.gen_opparam_buf[s->latest_tgt_parm_idx] = CHATKEY_CMP_EQUALITY; ++ tcg_ctx.gen_opparam_buf[s->latest_tgt_parm_idx] = ECLIPSER_CMP_EQUALITY; + cc_optimize_flag = 0; + } else if ((jcc_op == JCC_B || jcc_op == JCC_BE) && s->latest_tgt_parm_idx != -1) { -+ tcg_ctx.gen_opparam_buf[s->latest_tgt_parm_idx] = CHATKEY_CMP_SIZE_UNSIGNED; ++ tcg_ctx.gen_opparam_buf[s->latest_tgt_parm_idx] = ECLIPSER_CMP_SIZE_UNSIGNED; + cc_optimize_flag = 0; + } else if ((jcc_op == JCC_L || jcc_op == JCC_LE) && s->latest_tgt_parm_idx != -1) { -+ tcg_ctx.gen_opparam_buf[s->latest_tgt_parm_idx] = CHATKEY_CMP_SIZE_SIGNED; ++ tcg_ctx.gen_opparam_buf[s->latest_tgt_parm_idx] = ECLIPSER_CMP_SIZE_SIGNED; + cc_optimize_flag = 0; + } + @@ -145,8 +145,8 @@ } else { tcg_gen_mov_tl(cpu_cc_srcT, cpu_T0); - tcg_gen_sub_tl(cpu_T0, cpu_T0, cpu_T1); -+ if (chatkey_EP_passed) { -+ tcg_gen_sub_tl_chatkey(cpu_T0, cpu_T0, cpu_T1, ot); ++ if (eclipser_EP_passed) { ++ tcg_gen_sub_tl_eclipser(cpu_T0, cpu_T0, cpu_T1, ot); + // Record the index of comparison type argument. + assert(tcg_ctx.gen_next_parm_idx >= 2); + s1->latest_tgt_parm_idx = tcg_ctx.gen_next_parm_idx - 2; @@ -185,8 +185,8 @@ tcg_gen_mov_tl(cpu_cc_src, cpu_T1); tcg_gen_mov_tl(cpu_cc_srcT, cpu_T0); - tcg_gen_sub_tl(cpu_cc_dst, cpu_T0, cpu_T1); -+ if (chatkey_EP_passed) { -+ tcg_gen_sub_tl_chatkey(cpu_cc_dst, cpu_T0, cpu_T1, ot); ++ if (eclipser_EP_passed) { ++ tcg_gen_sub_tl_eclipser(cpu_cc_dst, cpu_T0, cpu_T1, ot); + // Record the index of comparison type argument. + assert(tcg_ctx.gen_next_parm_idx >= 2); + s1->latest_tgt_parm_idx = tcg_ctx.gen_next_parm_idx - 2; @@ -233,8 +233,8 @@ + /* We only focus on test instruction with two same registers as operand, + * e.g. "test eax, eax". + */ -+ if (reg == rm && chatkey_EP_passed) { -+ gen_op_testl_T0_T1_cc_chatkey(ot); ++ if (reg == rm && eclipser_EP_passed) { ++ gen_op_testl_T0_T1_cc_eclipser(ot); + // Record the index of comparison type argument. + assert(tcg_ctx.gen_next_parm_idx >= 2); + s->latest_tgt_parm_idx = tcg_ctx.gen_next_parm_idx - 2; @@ -252,17 +252,17 @@ + jcc_op = (b >> 1) & 7; + cc_optimize_flag = 1; + /* If this is a SETCC instruction followed by sub/cmp/test, replace the -+ * previous comparison type 'CHATKEY_IGNORE' with a new one, to indicate -+ * that this is our instrumentation target. ++ * previous comparison type 'ECLIPSER_IGNORE' with a new one, to ++ * indicate that this is our instrumentation target. + */ + if (jcc_op == JCC_Z && s->latest_tgt_parm_idx != -1) { -+ tcg_ctx.gen_opparam_buf[s->latest_tgt_parm_idx] = CHATKEY_CMP_EQUALITY; ++ tcg_ctx.gen_opparam_buf[s->latest_tgt_parm_idx] = ECLIPSER_CMP_EQUALITY; + cc_optimize_flag = 0; + } else if ((jcc_op == JCC_B || jcc_op == JCC_BE) && s->latest_tgt_parm_idx != -1) { -+ tcg_ctx.gen_opparam_buf[s->latest_tgt_parm_idx] = CHATKEY_CMP_SIZE_UNSIGNED; ++ tcg_ctx.gen_opparam_buf[s->latest_tgt_parm_idx] = ECLIPSER_CMP_SIZE_UNSIGNED; + cc_optimize_flag = 0; + } else if ((jcc_op == JCC_L || jcc_op == JCC_LE) && s->latest_tgt_parm_idx != -1) { -+ tcg_ctx.gen_opparam_buf[s->latest_tgt_parm_idx] = CHATKEY_CMP_SIZE_SIGNED; ++ tcg_ctx.gen_opparam_buf[s->latest_tgt_parm_idx] = ECLIPSER_CMP_SIZE_SIGNED; + cc_optimize_flag = 0; + } modrm = cpu_ldub_code(env, s->pc++); @@ -274,17 +274,17 @@ + jcc_op = (b >> 1) & 7; + cc_optimize_flag = 1; + /* If this is a CMOV instruction followed by sub/cmp/test, replace the -+ * previous comparison type 'CHATKEY_IGNORE' with a new one, to indicate -+ * that this is our instrumentation target. ++ * previous comparison type 'ECLIPSER_IGNORE' with a new one, to ++ * indicate that this is our instrumentation target. + */ + if (jcc_op == JCC_Z && s->latest_tgt_parm_idx != -1) { -+ tcg_ctx.gen_opparam_buf[s->latest_tgt_parm_idx] = CHATKEY_CMP_EQUALITY; ++ tcg_ctx.gen_opparam_buf[s->latest_tgt_parm_idx] = ECLIPSER_CMP_EQUALITY; + cc_optimize_flag = 0; + } else if ((jcc_op == JCC_B || jcc_op == JCC_BE) && s->latest_tgt_parm_idx != -1) { -+ tcg_ctx.gen_opparam_buf[s->latest_tgt_parm_idx] = CHATKEY_CMP_SIZE_UNSIGNED; ++ tcg_ctx.gen_opparam_buf[s->latest_tgt_parm_idx] = ECLIPSER_CMP_SIZE_UNSIGNED; + cc_optimize_flag = 0; + } else if ((jcc_op == JCC_L || jcc_op == JCC_LE) && s->latest_tgt_parm_idx != -1) { -+ tcg_ctx.gen_opparam_buf[s->latest_tgt_parm_idx] = CHATKEY_CMP_SIZE_SIGNED; ++ tcg_ctx.gen_opparam_buf[s->latest_tgt_parm_idx] = ECLIPSER_CMP_SIZE_SIGNED; + cc_optimize_flag = 0; + } if (!(s->cpuid_features & CPUID_CMOV)) { diff --git a/Instrumentor/patches-common/elfload.diff b/Instrumentor/patches-common/elfload.diff index c88ca7c..c7bfe2d 100644 --- a/Instrumentor/patches-common/elfload.diff +++ b/Instrumentor/patches-common/elfload.diff @@ -4,7 +4,7 @@ #define ELF_OSABI ELFOSABI_SYSV -+extern abi_ulong chatkey_entry_point; ++extern abi_ulong eclipser_entry_point; + /* from personality.h */ @@ -13,7 +13,7 @@ info->brk = 0; info->elf_flags = ehdr->e_flags; -+ if (!chatkey_entry_point) chatkey_entry_point = info->entry; ++ if (!eclipser_entry_point) eclipser_entry_point = info->entry; + for (i = 0; i < ehdr->e_phnum; i++) { struct elf_phdr *eppnt = phdr + i; diff --git a/Instrumentor/patches-common/signal.diff b/Instrumentor/patches-common/signal.diff index 950918c..3fa60e3 100644 --- a/Instrumentor/patches-common/signal.diff +++ b/Instrumentor/patches-common/signal.diff @@ -4,7 +4,7 @@ #include "target_signal.h" #include "trace.h" -+extern void chatkey_exit(void); ++extern void eclipser_exit(void); + static struct target_sigaltstack target_sigaltstack_used = { .ss_sp = 0, @@ -13,12 +13,12 @@ struct target_sigaction *sa; TaskState *ts = cpu->opaque; -+ /* If the signal indicates a crash or timeout, call chatkey_exit() to flush ++ /* If the signal indicates a crash or timeout, call eclipser_exit() to flush + * out the information traced until now. + */ + if (sig == SIGSEGV || sig == SIGFPE || sig == SIGILL || sig == SIGABRT || + sig == SIGTERM) { -+ chatkey_exit(); ++ eclipser_exit(); + } + trace_user_handle_signal(cpu_env, sig); diff --git a/Instrumentor/patches-coverage/afl-qemu-cpu-inl.h b/Instrumentor/patches-coverage/afl-qemu-cpu-inl.h index 758b455..c8d9177 100644 --- a/Instrumentor/patches-coverage/afl-qemu-cpu-inl.h +++ b/Instrumentor/patches-coverage/afl-qemu-cpu-inl.h @@ -53,7 +53,7 @@ static void afl_forkserver(CPUState *cpu) { static unsigned char tmp[4]; - if (atoi(getenv("CK_FORK_SERVER")) != 1) return; + if (atoi(getenv("ECL_FORK_SERVER")) != 1) return; /* Tell the parent that we're alive. If the parent doesn't want to talk, assume that we're not running in forkserver mode. */ diff --git a/Instrumentor/patches-coverage/cpu-exec.diff b/Instrumentor/patches-coverage/cpu-exec.diff index cc75516..228a0dc 100644 --- a/Instrumentor/patches-coverage/cpu-exec.diff +++ b/Instrumentor/patches-coverage/cpu-exec.diff @@ -5,10 +5,10 @@ #include "sysemu/replay.h" +#include "afl-qemu-cpu-inl.h" -+extern abi_ulong chatkey_entry_point; /* ELF entry point (_start) */ -+extern void chatkey_setup_before_forkserver(void); -+extern void chatkey_setup_after_forkserver(void); -+extern void chatkey_log_bb(abi_ulong addr); ++extern abi_ulong eclipser_entry_point; /* ELF entry point (_start) */ ++extern void eclipser_setup_before_forkserver(void); ++extern void eclipser_setup_after_forkserver(void); ++extern void eclipser_log_bb(abi_ulong addr); + /* -icount align implementation. */ @@ -20,14 +20,14 @@ + abi_ulong entry_pc; + + entry_pc = itb->pc; -+ if(entry_pc == chatkey_entry_point) { -+ chatkey_setup_before_forkserver(); ++ if(entry_pc == eclipser_entry_point) { ++ eclipser_setup_before_forkserver(); + // Resolves util/rcu.c assertion error issue (cf. AFL-2.53b). + rcu_disable_atfork(); + afl_forkserver(cpu); -+ chatkey_setup_after_forkserver(); ++ eclipser_setup_after_forkserver(); + } -+ chatkey_log_bb(entry_pc); ++ eclipser_log_bb(entry_pc); qemu_log_mask_and_addr(CPU_LOG_EXEC, itb->pc, "Trace %p [%d: " TARGET_FMT_lx "] %s\n", diff --git a/Instrumentor/patches-coverage/chatkey.c b/Instrumentor/patches-coverage/eclipser.c similarity index 81% rename from Instrumentor/patches-coverage/chatkey.c rename to Instrumentor/patches-coverage/eclipser.c index b920f4b..5b4d28c 100644 --- a/Instrumentor/patches-coverage/chatkey.c +++ b/Instrumentor/patches-coverage/eclipser.c @@ -19,13 +19,13 @@ extern unsigned int afl_forksrv_pid; #define FORKSRV_FD 198 #define TSL_FD (FORKSRV_FD - 1) -void chatkey_setup_before_forkserver(void); -void chatkey_setup_after_forkserver(void); -void chatkey_close_fp(void); -void chatkey_exit(void); -void chatkey_log_bb(abi_ulong addr); +void eclipser_setup_before_forkserver(void); +void eclipser_setup_after_forkserver(void); +void eclipser_detach(void); +void eclipser_exit(void); +void eclipser_log_bb(abi_ulong addr); -abi_ulong chatkey_entry_point; /* ELF entry point (_start) */ +abi_ulong eclipser_entry_point; /* ELF entry point (_start) */ static char * coverage_path = NULL; static char * dbg_path = NULL; @@ -37,19 +37,19 @@ static int found_new_edge = 0; static int found_new_path = 0; // TODO. Extend to measure path coverage, too. static unsigned char * edge_bitmap = NULL; -void chatkey_setup_before_forkserver(void) { - char * bitmap_path = getenv("CK_BITMAP_LOG"); +void eclipser_setup_before_forkserver(void) { + char * bitmap_path = getenv("ECL_BITMAP_LOG"); int bitmap_fd = open(bitmap_path, O_RDWR | O_CREAT, 0644); edge_bitmap = (unsigned char*) mmap(NULL, 0x10000, PROT_READ | PROT_WRITE, MAP_SHARED, bitmap_fd, 0); assert(edge_bitmap != (void *) -1); - coverage_path = getenv("CK_COVERAGE_LOG"); - dbg_path = getenv("CK_DBG_LOG"); + coverage_path = getenv("ECL_COVERAGE_LOG"); + dbg_path = getenv("ECL_DBG_LOG"); } -void chatkey_setup_after_forkserver(void) { +void eclipser_setup_after_forkserver(void) { /* Open file pointers and descriptors early, since if we try to open them in - * chatkey_exit(), it gets mixed with stderr & stdout stream. This seems to + * eclipser_exit(), it gets mixed with stderr & stdout stream. This seems to * be an issue due to incorrect file descriptor management in QEMU code. */ @@ -63,8 +63,9 @@ void chatkey_setup_after_forkserver(void) { } } -// When fork() syscall is encountered, child process should call this function. -void chatkey_close_fp(void) { +// When fork() syscall is encountered, child process should call this function +// to detach from Eclipser. +void eclipser_detach(void) { // Close file pointers, to avoid dumping log twice. if (coverage_fp) { fclose(coverage_fp); @@ -85,10 +86,10 @@ void chatkey_close_fp(void) { close(TSL_FD); } -void chatkey_exit(void) { +void eclipser_exit(void) { sigset_t mask; - // Block signals, since we register signal handler that calls chatkey_exit()/ + // Block signals, since we register signal handler that calls eclipser_exit()/ if (sigfillset(&mask) < 0) return; if (sigprocmask(SIG_BLOCK, &mask, NULL) < 0) @@ -111,7 +112,7 @@ void chatkey_exit(void) { } } -void chatkey_log_bb(abi_ulong addr) { +void eclipser_log_bb(abi_ulong addr) { abi_ulong prev_addr_local; abi_ulong edge, hash; unsigned int byte_idx, byte_mask; diff --git a/Instrumentor/patches-coverage/makefile-objs.diff b/Instrumentor/patches-coverage/makefile-objs.diff index 4e6545f..5d5ee39 100644 --- a/Instrumentor/patches-coverage/makefile-objs.diff +++ b/Instrumentor/patches-coverage/makefile-objs.diff @@ -4,4 +4,4 @@ obj-$(CONFIG_SOFTMMU) += tcg-all.o obj-$(CONFIG_SOFTMMU) += cputlb.o -obj-y += cpu-exec.o cpu-exec-common.o translate-all.o -+obj-y += cpu-exec.o cpu-exec-common.o translate-all.o chatkey.o ++obj-y += cpu-exec.o cpu-exec-common.o translate-all.o eclipser.o diff --git a/Instrumentor/patches-coverage/syscall.diff b/Instrumentor/patches-coverage/syscall.diff index e4f050e..93c3930 100644 --- a/Instrumentor/patches-coverage/syscall.diff +++ b/Instrumentor/patches-coverage/syscall.diff @@ -4,8 +4,8 @@ #include "qemu.h" -+extern void chatkey_exit(void); -+extern void chatkey_close_fp(void); ++extern void eclipser_exit(void); ++extern void eclipser_detach(void); +extern unsigned int afl_forksrv_pid; + #ifndef CLONE_IO @@ -15,7 +15,7 @@ ret = fork(); if (ret == 0) { /* Child Process. */ -+ chatkey_close_fp(); ++ eclipser_detach(); cpu_clone_regs(env, newsp); fork_end(1); /* There is a race condition here. The parent process could @@ -23,7 +23,7 @@ #ifdef TARGET_GPROF _mcleanup(); #endif -+ chatkey_exit(); ++ eclipser_exit(); gdb_exit(cpu_env, arg1); _exit(arg1); ret = 0; /* avoid warning */ @@ -31,7 +31,7 @@ #ifdef TARGET_GPROF _mcleanup(); #endif -+ chatkey_exit(); ++ eclipser_exit(); gdb_exit(cpu_env, arg1); ret = get_errno(exit_group(arg1)); break; diff --git a/Instrumentor/prepare_qemu.sh b/Instrumentor/prepare_qemu.sh index 35b0ff2..902d50f 100755 --- a/Instrumentor/prepare_qemu.sh +++ b/Instrumentor/prepare_qemu.sh @@ -21,7 +21,7 @@ QEMU_URL="https://download.qemu.org/qemu-${VERSION}.tar.bz2" QEMU_SHA384="9496d1d209d3a49d67dd83fbcf3f2bf376b7d5f500b0247d813639d104effa79d8a39e64f94d6f9541f6e9d8e3cc574f" echo "=========================================" -echo "Chatkey instrumentation QEMU build script" +echo "QEMU build script for Eclipser" echo "=========================================" echo @@ -34,7 +34,7 @@ if [ ! "`uname -s`" = "Linux" ]; then fi -if [ ! -f "patches-coverage/chatkey.c" -o ! -f "patches-branch/chatkey.c" ]; then +if [ ! -f "patches-coverage/eclipser.c" -o ! -f "patches-branch/eclipser.c" ]; then echo "[-] Error: key files not found - wrong working directory?" exit 1 @@ -128,7 +128,7 @@ cp -r "qemu-${VERSION}" "qemu-${VERSION}-bbcount" echo "[*] Applying patches for coverage..." cp patches-coverage/afl-qemu-cpu-inl.h qemu-${VERSION}-coverage/accel/tcg/ -cp patches-coverage/chatkey.c qemu-${VERSION}-coverage/accel/tcg/ +cp patches-coverage/eclipser.c qemu-${VERSION}-coverage/accel/tcg/ patch -p0 = 2 then - set_env("CK_DBG_LOG", System.IO.Path.GetFullPath(dbgLog)) + set_env("ECL_DBG_LOG", System.IO.Path.GetFullPath(dbgLog)) // Initialize C wrapper code. initialize_exec () // Disable TranslationBlock chaining feature of QEMU. set_env("QEMU_LOG", "nochain") // Initialize fork server. forkServerEnabled <- true - set_env("CK_FORK_SERVER", "1") + set_env("ECL_FORK_SERVER", "1") let cmdLine = opt.Arg.Split(WHITES, StringSplitOptions.RemoveEmptyEntries) let coverageTracer = selectTracer Coverage opt.Architecture let args = Array.append [|coverageTracer; opt.TargetProg|] cmdLine @@ -91,7 +91,7 @@ let cleanup () = let abandonForkServer () = log "Abandon fork server" forkServerEnabled <- false - set_env("CK_FORK_SERVER", "0") + set_env("ECL_FORK_SERVER", "0") kill_forkserver () (*** File handling utilities ***) @@ -197,9 +197,9 @@ let private runBranchTracerForked opt stdin addr idx covMeasure = signal let private setEnvForBranch (addr: uint64) (idx: uint32) covMeasure = - set_env("CK_FEED_ADDR", sprintf "%016x" addr) - set_env("CK_FEED_IDX", sprintf "%016x" idx) - set_env("CK_MEASURE_COV", sprintf "%d" (CoverageMeasure.toEnum covMeasure)) + set_env("ECL_BRANCH_ADDR", sprintf "%016x" addr) + set_env("ECL_BRANCH_IDX", sprintf "%016x" idx) + set_env("ECL_MEASURE_COV", sprintf "%d" (CoverageMeasure.toEnum covMeasure)) (*** Top-level tracer executor functions ***) From 4f31203c29669ea6349cf4848b8dd41a49021574 Mon Sep 17 00:00:00 2001 From: Jaeseung Choi Date: Sun, 11 Oct 2020 23:44:37 -0700 Subject: [PATCH 18/29] Fix QEMU instrumentation logic to reintroduce TB chaining --- Instrumentor/generate_branch_patch.sh | 8 +++- Instrumentor/generate_coverage_patch.sh | 12 +++++ .../patches-branch/afl-qemu-cpu-inl.h | 44 ++++++++++++++++--- Instrumentor/patches-branch/cpu-exec.diff | 39 ++++++++++++---- Instrumentor/patches-branch/eclipser.c | 4 +- .../patches-branch/makefile-target.diff | 4 +- Instrumentor/patches-branch/optimize.diff | 4 +- Instrumentor/patches-branch/syscall.diff | 4 +- .../patches-branch/target-helper.diff | 12 +++++ .../{translate.diff => target-translate.diff} | 20 +++++++-- Instrumentor/patches-branch/tcg-op.diff | 4 +- Instrumentor/patches-branch/tcg-opc.diff | 4 +- Instrumentor/patches-branch/tcg-target.diff | 4 +- Instrumentor/patches-branch/tcg.diff | 18 -------- .../patches-coverage/afl-qemu-cpu-inl.h | 44 ++++++++++++++++--- Instrumentor/patches-coverage/cpu-exec.diff | 39 ++++++++++++---- Instrumentor/patches-coverage/eclipser.c | 4 +- .../patches-coverage/makefile-objs.diff | 4 +- Instrumentor/patches-coverage/syscall.diff | 4 +- .../patches-coverage/target-helper.diff | 12 +++++ .../patches-coverage/target-translate.diff | 16 +++++++ Instrumentor/prepare_qemu.sh | 5 ++- Instrumentor/repatch.sh | 8 +++- src/Core/Executor.fs | 2 - 24 files changed, 247 insertions(+), 72 deletions(-) create mode 100644 Instrumentor/patches-branch/target-helper.diff rename Instrumentor/patches-branch/{translate.diff => target-translate.diff} (95%) delete mode 100644 Instrumentor/patches-branch/tcg.diff create mode 100644 Instrumentor/patches-coverage/target-helper.diff create mode 100644 Instrumentor/patches-coverage/target-translate.diff diff --git a/Instrumentor/generate_branch_patch.sh b/Instrumentor/generate_branch_patch.sh index 4fcc3e8..9e759e0 100755 --- a/Instrumentor/generate_branch_patch.sh +++ b/Instrumentor/generate_branch_patch.sh @@ -50,10 +50,16 @@ diff -Naur qemu-${VERSION}-branch/tcg/i386/tcg-target.inc.c.orig \ qemu-${VERSION}-branch/tcg/i386/tcg-target.inc.c \ > patches-branch/tcg-target.diff +cp qemu-${VERSION}/target/i386/helper.h \ + qemu-${VERSION}-branch/target/i386/helper.h.orig +diff -Naur qemu-${VERSION}-branch/target/i386/helper.h.orig \ + qemu-${VERSION}-branch/target/i386/helper.h \ + > patches-branch/target-helper.diff + cp qemu-${VERSION}/target/i386/translate.c \ qemu-${VERSION}-branch/target/i386/translate.c.orig diff -Naur qemu-${VERSION}-branch/target/i386/translate.c.orig \ qemu-${VERSION}-branch/target/i386/translate.c \ - > patches-branch/translate.diff + > patches-branch/target-translate.diff rm -rf qemu-${VERSION}-branch diff --git a/Instrumentor/generate_coverage_patch.sh b/Instrumentor/generate_coverage_patch.sh index edec52f..e20ba7f 100755 --- a/Instrumentor/generate_coverage_patch.sh +++ b/Instrumentor/generate_coverage_patch.sh @@ -26,4 +26,16 @@ diff -Naur qemu-${VERSION}-coverage/linux-user/syscall.c.orig \ qemu-${VERSION}-coverage/linux-user/syscall.c \ > patches-coverage/syscall.diff +cp qemu-${VERSION}/target/i386/helper.h \ + qemu-${VERSION}-coverage/target/i386/helper.h.orig +diff -Naur qemu-${VERSION}-coverage/target/i386/helper.h.orig \ + qemu-${VERSION}-coverage/target/i386/helper.h \ + > patches-coverage/target-helper.diff + +cp qemu-${VERSION}/target/i386/translate.c \ + qemu-${VERSION}-coverage/target/i386/translate.c.orig +diff -Naur qemu-${VERSION}-coverage/target/i386/translate.c.orig \ + qemu-${VERSION}-coverage/target/i386/translate.c \ + > patches-coverage/target-translate.diff + rm -rf qemu-${VERSION}-coverage diff --git a/Instrumentor/patches-branch/afl-qemu-cpu-inl.h b/Instrumentor/patches-branch/afl-qemu-cpu-inl.h index f9de5f2..4e897fb 100644 --- a/Instrumentor/patches-branch/afl-qemu-cpu-inl.h +++ b/Instrumentor/patches-branch/afl-qemu-cpu-inl.h @@ -37,7 +37,8 @@ unsigned int afl_forksrv_pid; static void afl_forkserver(CPUState*); static void afl_wait_tsl(CPUState*, int); -static void afl_request_tsl(target_ulong, target_ulong, uint64_t); +static void afl_request_tsl(target_ulong, target_ulong, uint64_t, + TranslationBlock*, int); /* Data structure passed around by the translate handlers: */ @@ -45,6 +46,14 @@ struct afl_tsl { target_ulong pc; target_ulong cs_base; uint64_t flags; + int chained; +}; + +struct afl_chain { + target_ulong last_pc; + target_ulong last_cs_base; + uint64_t last_flags; + int tb_exit; }; /************************* @@ -126,19 +135,30 @@ static void afl_forkserver(CPUState *cpu) { we tell the parent to mirror the operation, so that the next fork() has a cached copy. */ -static void afl_request_tsl(target_ulong pc, target_ulong cb, uint64_t flags) { - +static void afl_request_tsl(target_ulong pc, target_ulong cb, uint64_t flags, + TranslationBlock* last_tb, int tb_exit) { struct afl_tsl t; + struct afl_chain c; if (!afl_fork_child) return; t.pc = pc; t.cs_base = cb; t.flags = flags; + t.chained = (last_tb != NULL); if (write(TSL_FD, &t, sizeof(struct afl_tsl)) != sizeof(struct afl_tsl)) return; + if (last_tb) { + c.last_pc = last_tb->pc; + c.last_cs_base = last_tb->cs_base; + c.last_flags = last_tb->flags; + c.tb_exit = tb_exit; + + if (write(TSL_FD, &c, sizeof(struct afl_chain)) != sizeof(struct afl_chain)) + return; + } } /* This is the other side of the same channel. Since timeouts are handled by @@ -147,7 +167,8 @@ static void afl_request_tsl(target_ulong pc, target_ulong cb, uint64_t flags) { static void afl_wait_tsl(CPUState *cpu, int fd) { struct afl_tsl t; - TranslationBlock *tb; + struct afl_chain c; + TranslationBlock *tb, *last_tb; while (1) { @@ -161,11 +182,24 @@ static void afl_wait_tsl(CPUState *cpu, int fd) { if(!tb) { mmap_lock(); tb_lock(); - tb_gen_code(cpu, t.pc, t.cs_base, t.flags, 0); + tb = tb_gen_code(cpu, t.pc, t.cs_base, t.flags, 0); mmap_unlock(); tb_unlock(); } + if (t.chained) { + if (read(fd, &c, sizeof(struct afl_chain)) != sizeof(struct afl_chain)) + break; + + last_tb = tb_htable_lookup(cpu, c.last_pc, c.last_cs_base, c.last_flags); + if (last_tb) { + tb_lock(); + if (!tb->invalid) { + tb_add_jump(last_tb, c.tb_exit, tb); + } + tb_unlock(); + } + } } close(fd); diff --git a/Instrumentor/patches-branch/cpu-exec.diff b/Instrumentor/patches-branch/cpu-exec.diff index 0d1d5b0..0f4439b 100644 --- a/Instrumentor/patches-branch/cpu-exec.diff +++ b/Instrumentor/patches-branch/cpu-exec.diff @@ -1,6 +1,6 @@ ---- qemu-2.10.0-branch/accel/tcg/cpu-exec.c.orig 2020-10-03 21:06:01.813430015 -0700 -+++ qemu-2.10.0-branch/accel/tcg/cpu-exec.c 2020-10-03 21:05:57.233402733 -0700 -@@ -36,6 +36,12 @@ +--- qemu-2.10.0-branch/accel/tcg/cpu-exec.c.orig 2020-10-12 02:23:49.165865730 -0700 ++++ qemu-2.10.0-branch/accel/tcg/cpu-exec.c 2020-10-12 02:23:48.489872694 -0700 +@@ -36,6 +36,11 @@ #include "sysemu/cpus.h" #include "sysemu/replay.h" @@ -8,12 +8,11 @@ +extern abi_ulong eclipser_entry_point; /* ELF entry point (_start) */ +extern void eclipser_setup_before_forkserver(void); +extern void eclipser_setup_after_forkserver(void); -+extern void eclipser_log_bb(abi_ulong addr); + /* -icount align implementation. */ typedef struct SyncClocks { -@@ -143,6 +149,17 @@ +@@ -143,6 +148,16 @@ TranslationBlock *last_tb; int tb_exit; uint8_t *tb_ptr = itb->tc_ptr; @@ -27,15 +26,39 @@ + afl_forkserver(cpu); + eclipser_setup_after_forkserver(); + } -+ eclipser_log_bb(entry_pc); qemu_log_mask_and_addr(CPU_LOG_EXEC, itb->pc, "Trace %p [%d: " TARGET_FMT_lx "] %s\n", -@@ -365,6 +382,7 @@ +@@ -337,7 +352,7 @@ + TranslationBlock *tb; + target_ulong cs_base, pc; + uint32_t flags; +- bool have_tb_lock = false; ++ bool have_tb_lock = false, translated = false, chained = false; + + /* we record a subset of the CPU state. It will + always be the same before a given translated block +@@ -365,6 +380,7 @@ if (!tb) { /* if no translated code available, then translate it now */ tb = tb_gen_code(cpu, pc, cs_base, flags, 0); -+ afl_request_tsl(pc, cs_base, flags); ++ translated = true; } mmap_unlock(); +@@ -390,11 +406,15 @@ + } + if (!tb->invalid) { + tb_add_jump(last_tb, tb_exit, tb); ++ chained = true; + } + } + if (have_tb_lock) { + tb_unlock(); + } ++ if (translated || chained) { ++ afl_request_tsl(pc, cs_base, flags, chained ? last_tb : NULL, tb_exit); ++ } + return tb; + } + diff --git a/Instrumentor/patches-branch/eclipser.c b/Instrumentor/patches-branch/eclipser.c index d0f88b0..27b6b59 100644 --- a/Instrumentor/patches-branch/eclipser.c +++ b/Instrumentor/patches-branch/eclipser.c @@ -34,7 +34,7 @@ void eclipser_setup_after_forkserver(void); void eclipser_detach(void); void eclipser_exit(void); void eclipser_log_branch(abi_ulong oprnd1, abi_ulong oprnd2, unsigned char type); -void eclipser_log_bb(abi_ulong addr); +void helper_eclipser_log_bb(abi_ulong addr); abi_ulong eclipser_entry_point = 0; /* ELF entry point (_start) */ abi_ulong eclipser_curr_addr = 0; @@ -306,7 +306,7 @@ void eclipser_log_branch(abi_ulong oprnd1, abi_ulong oprnd2, unsigned char type) } } -void eclipser_log_bb(abi_ulong addr) { +void helper_eclipser_log_bb(abi_ulong addr) { abi_ulong prev_addr_local; abi_ulong edge, hash; unsigned int byte_idx, byte_mask; diff --git a/Instrumentor/patches-branch/makefile-target.diff b/Instrumentor/patches-branch/makefile-target.diff index 8ccea59..c2fdfd1 100644 --- a/Instrumentor/patches-branch/makefile-target.diff +++ b/Instrumentor/patches-branch/makefile-target.diff @@ -1,5 +1,5 @@ ---- qemu-2.10.0-branch/Makefile.target.orig 2020-10-03 21:06:01.957430875 -0700 -+++ qemu-2.10.0-branch/Makefile.target 2020-10-03 21:05:57.237402757 -0700 +--- qemu-2.10.0-branch/Makefile.target.orig 2020-10-12 02:23:49.173865649 -0700 ++++ qemu-2.10.0-branch/Makefile.target 2020-10-12 02:23:49.109866308 -0700 @@ -94,7 +94,7 @@ obj-y += exec.o obj-y += accel/ diff --git a/Instrumentor/patches-branch/optimize.diff b/Instrumentor/patches-branch/optimize.diff index c5c9120..f6be443 100644 --- a/Instrumentor/patches-branch/optimize.diff +++ b/Instrumentor/patches-branch/optimize.diff @@ -1,5 +1,5 @@ ---- qemu-2.10.0-branch/tcg/optimize.c.orig 2020-10-03 21:06:01.973430971 -0700 -+++ qemu-2.10.0-branch/tcg/optimize.c 2020-10-03 21:06:00.237420601 -0700 +--- qemu-2.10.0-branch/tcg/optimize.c.orig 2020-10-12 02:23:49.177865607 -0700 ++++ qemu-2.10.0-branch/tcg/optimize.c 2020-10-12 02:23:49.089866514 -0700 @@ -678,13 +678,15 @@ continue; } diff --git a/Instrumentor/patches-branch/syscall.diff b/Instrumentor/patches-branch/syscall.diff index cb34aad..a863558 100644 --- a/Instrumentor/patches-branch/syscall.diff +++ b/Instrumentor/patches-branch/syscall.diff @@ -1,5 +1,5 @@ ---- qemu-2.10.0-branch/linux-user/syscall.c.orig 2020-10-03 21:06:01.965430923 -0700 -+++ qemu-2.10.0-branch/linux-user/syscall.c 2020-10-03 21:05:57.281403019 -0700 +--- qemu-2.10.0-branch/linux-user/syscall.c.orig 2020-10-12 02:23:49.173865649 -0700 ++++ qemu-2.10.0-branch/linux-user/syscall.c 2020-10-12 02:23:48.497872612 -0700 @@ -116,6 +116,10 @@ #include "qemu.h" diff --git a/Instrumentor/patches-branch/target-helper.diff b/Instrumentor/patches-branch/target-helper.diff new file mode 100644 index 0000000..60532af --- /dev/null +++ b/Instrumentor/patches-branch/target-helper.diff @@ -0,0 +1,12 @@ +--- qemu-2.10.0-branch/target/i386/helper.h.orig 2020-10-12 02:23:49.185865526 -0700 ++++ qemu-2.10.0-branch/target/i386/helper.h 2020-10-12 02:23:49.057866843 -0700 +@@ -226,3 +226,9 @@ + DEF_HELPER_3(rclq, tl, env, tl, tl) + DEF_HELPER_3(rcrq, tl, env, tl, tl) + #endif ++ ++#ifdef TARGET_X86_64 ++DEF_HELPER_1(eclipser_log_bb, void, i64) ++#else ++DEF_HELPER_1(eclipser_log_bb, void, i32) ++#endif diff --git a/Instrumentor/patches-branch/translate.diff b/Instrumentor/patches-branch/target-translate.diff similarity index 95% rename from Instrumentor/patches-branch/translate.diff rename to Instrumentor/patches-branch/target-translate.diff index 83bac2e..872579c 100644 --- a/Instrumentor/patches-branch/translate.diff +++ b/Instrumentor/patches-branch/target-translate.diff @@ -1,5 +1,5 @@ ---- qemu-2.10.0-branch/target/i386/translate.c.orig 2020-10-03 21:06:01.989431067 -0700 -+++ qemu-2.10.0-branch/target/i386/translate.c 2020-10-03 21:05:57.213402615 -0700 +--- qemu-2.10.0-branch/target/i386/translate.c.orig 2020-10-12 02:23:49.185865526 -0700 ++++ qemu-2.10.0-branch/target/i386/translate.c 2020-10-12 02:23:49.057866843 -0700 @@ -71,6 +71,10 @@ //#define MACRO_TEST 1 @@ -298,7 +298,21 @@ break; /************************/ -@@ -8445,6 +8553,9 @@ +@@ -8390,6 +8498,13 @@ + int num_insns; + int max_insns; + ++#ifdef TARGET_X86_64 ++ TCGv_i64 pc_var = tcg_const_i64((uint64_t)tb->pc); ++#else ++ TCGv_i32 pc_var = tcg_const_i32((uint64_t)tb->pc); ++#endif ++ gen_helper_eclipser_log_bb(pc_var); ++ + /* generate intermediate code */ + pc_start = tb->pc; + cs_base = tb->cs_base; +@@ -8445,6 +8560,9 @@ printf("ERROR addseg\n"); #endif diff --git a/Instrumentor/patches-branch/tcg-op.diff b/Instrumentor/patches-branch/tcg-op.diff index e001c84..23a7693 100644 --- a/Instrumentor/patches-branch/tcg-op.diff +++ b/Instrumentor/patches-branch/tcg-op.diff @@ -1,5 +1,5 @@ ---- qemu-2.10.0-branch/tcg/tcg-op.h.orig 2020-10-03 21:06:01.977430995 -0700 -+++ qemu-2.10.0-branch/tcg/tcg-op.h 2020-10-03 21:06:00.237420601 -0700 +--- qemu-2.10.0-branch/tcg/tcg-op.h.orig 2020-10-12 02:23:49.177865607 -0700 ++++ qemu-2.10.0-branch/tcg/tcg-op.h 2020-10-12 02:23:49.089866514 -0700 @@ -26,6 +26,11 @@ #include "exec/helper-proto.h" #include "exec/helper-gen.h" diff --git a/Instrumentor/patches-branch/tcg-opc.diff b/Instrumentor/patches-branch/tcg-opc.diff index 01b4ce7..fb9452e 100644 --- a/Instrumentor/patches-branch/tcg-opc.diff +++ b/Instrumentor/patches-branch/tcg-opc.diff @@ -1,5 +1,5 @@ ---- qemu-2.10.0-branch/tcg/tcg-opc.h.orig 2020-10-03 21:06:01.981431019 -0700 -+++ qemu-2.10.0-branch/tcg/tcg-opc.h 2020-10-03 21:06:00.237420601 -0700 +--- qemu-2.10.0-branch/tcg/tcg-opc.h.orig 2020-10-12 02:23:49.177865607 -0700 ++++ qemu-2.10.0-branch/tcg/tcg-opc.h 2020-10-12 02:23:49.089866514 -0700 @@ -59,7 +59,7 @@ DEF(st_i32, 0, 2, 1, 0) /* arith */ diff --git a/Instrumentor/patches-branch/tcg-target.diff b/Instrumentor/patches-branch/tcg-target.diff index af9de98..f176280 100644 --- a/Instrumentor/patches-branch/tcg-target.diff +++ b/Instrumentor/patches-branch/tcg-target.diff @@ -1,5 +1,5 @@ ---- qemu-2.10.0-branch/tcg/i386/tcg-target.inc.c.orig 2020-10-03 21:06:01.985431043 -0700 -+++ qemu-2.10.0-branch/tcg/i386/tcg-target.inc.c 2020-10-03 21:06:00.241420625 -0700 +--- qemu-2.10.0-branch/tcg/i386/tcg-target.inc.c.orig 2020-10-12 02:23:49.181865568 -0700 ++++ qemu-2.10.0-branch/tcg/i386/tcg-target.inc.c 2020-10-12 02:23:49.089866514 -0700 @@ -24,6 +24,10 @@ #include "tcg-be-ldst.h" diff --git a/Instrumentor/patches-branch/tcg.diff b/Instrumentor/patches-branch/tcg.diff deleted file mode 100644 index 5f81399..0000000 --- a/Instrumentor/patches-branch/tcg.diff +++ /dev/null @@ -1,18 +0,0 @@ ---- qemu-2.3.0-branch/tcg/tcg.h.orig 2017-08-18 10:12:58.561955139 +0900 -+++ qemu-2.3.0-branch/tcg/tcg.h 2017-08-18 10:12:44.066164190 +0900 -@@ -29,6 +29,15 @@ - #include "qemu/bitops.h" - #include "tcg-target.h" - -+/* Constants used for Eclipser instrumentation. Specifies whether an instruction -+ * is target of Eclipser instrumentation (i.e. decides whether to insert code -+ * that records Eflags register value). -+ */ -+#define ECLIPSER_CMP_EQUALITY 0 -+#define ECLIPSER_CMP_SIZE_SIGNED 1 -+#define ECLIPSER_CMP_SIZE_UNSIGNED 2 -+#define ECLIPSER_IGNORE 3 -+ - /* Default target word size to pointer size. */ - #ifndef TCG_TARGET_REG_BITS - # if UINTPTR_MAX == UINT32_MAX diff --git a/Instrumentor/patches-coverage/afl-qemu-cpu-inl.h b/Instrumentor/patches-coverage/afl-qemu-cpu-inl.h index c8d9177..1cbfd08 100644 --- a/Instrumentor/patches-coverage/afl-qemu-cpu-inl.h +++ b/Instrumentor/patches-coverage/afl-qemu-cpu-inl.h @@ -33,7 +33,8 @@ unsigned int afl_forksrv_pid; static void afl_forkserver(CPUState*); static void afl_wait_tsl(CPUState*, int); -static void afl_request_tsl(target_ulong, target_ulong, uint64_t); +static void afl_request_tsl(target_ulong, target_ulong, uint64_t, + TranslationBlock*, int); /* Data structure passed around by the translate handlers: */ @@ -41,6 +42,14 @@ struct afl_tsl { target_ulong pc; target_ulong cs_base; uint64_t flags; + int chained; +}; + +struct afl_chain { + target_ulong last_pc; + target_ulong last_cs_base; + uint64_t last_flags; + int tb_exit; }; /************************* @@ -118,19 +127,30 @@ static void afl_forkserver(CPUState *cpu) { we tell the parent to mirror the operation, so that the next fork() has a cached copy. */ -static void afl_request_tsl(target_ulong pc, target_ulong cb, uint64_t flags) { - +static void afl_request_tsl(target_ulong pc, target_ulong cb, uint64_t flags, + TranslationBlock* last_tb, int tb_exit) { struct afl_tsl t; + struct afl_chain c; if (!afl_fork_child) return; t.pc = pc; t.cs_base = cb; t.flags = flags; + t.chained = (last_tb != NULL); if (write(TSL_FD, &t, sizeof(struct afl_tsl)) != sizeof(struct afl_tsl)) return; + if (last_tb) { + c.last_pc = last_tb->pc; + c.last_cs_base = last_tb->cs_base; + c.last_flags = last_tb->flags; + c.tb_exit = tb_exit; + + if (write(TSL_FD, &c, sizeof(struct afl_chain)) != sizeof(struct afl_chain)) + return; + } } /* This is the other side of the same channel. Since timeouts are handled by @@ -139,7 +159,8 @@ static void afl_request_tsl(target_ulong pc, target_ulong cb, uint64_t flags) { static void afl_wait_tsl(CPUState *cpu, int fd) { struct afl_tsl t; - TranslationBlock *tb; + struct afl_chain c; + TranslationBlock *tb, *last_tb; while (1) { @@ -153,11 +174,24 @@ static void afl_wait_tsl(CPUState *cpu, int fd) { if(!tb) { mmap_lock(); tb_lock(); - tb_gen_code(cpu, t.pc, t.cs_base, t.flags, 0); + tb = tb_gen_code(cpu, t.pc, t.cs_base, t.flags, 0); mmap_unlock(); tb_unlock(); } + if (t.chained) { + if (read(fd, &c, sizeof(struct afl_chain)) != sizeof(struct afl_chain)) + break; + + last_tb = tb_htable_lookup(cpu, c.last_pc, c.last_cs_base, c.last_flags); + if (last_tb) { + tb_lock(); + if (!tb->invalid) { + tb_add_jump(last_tb, c.tb_exit, tb); + } + tb_unlock(); + } + } } close(fd); diff --git a/Instrumentor/patches-coverage/cpu-exec.diff b/Instrumentor/patches-coverage/cpu-exec.diff index 228a0dc..8782ff2 100644 --- a/Instrumentor/patches-coverage/cpu-exec.diff +++ b/Instrumentor/patches-coverage/cpu-exec.diff @@ -1,6 +1,6 @@ ---- qemu-2.10.0-coverage/accel/tcg/cpu-exec.c.orig 2020-10-03 21:05:29.913244803 -0700 -+++ qemu-2.10.0-coverage/accel/tcg/cpu-exec.c 2020-10-03 21:05:21.857199845 -0700 -@@ -36,6 +36,12 @@ +--- qemu-2.10.0-coverage/accel/tcg/cpu-exec.c.orig 2020-10-12 02:23:45.417904334 -0700 ++++ qemu-2.10.0-coverage/accel/tcg/cpu-exec.c 2020-10-12 02:23:41.501944668 -0700 +@@ -36,6 +36,11 @@ #include "sysemu/cpus.h" #include "sysemu/replay.h" @@ -8,12 +8,11 @@ +extern abi_ulong eclipser_entry_point; /* ELF entry point (_start) */ +extern void eclipser_setup_before_forkserver(void); +extern void eclipser_setup_after_forkserver(void); -+extern void eclipser_log_bb(abi_ulong addr); + /* -icount align implementation. */ typedef struct SyncClocks { -@@ -143,6 +149,17 @@ +@@ -143,6 +148,16 @@ TranslationBlock *last_tb; int tb_exit; uint8_t *tb_ptr = itb->tc_ptr; @@ -27,15 +26,39 @@ + afl_forkserver(cpu); + eclipser_setup_after_forkserver(); + } -+ eclipser_log_bb(entry_pc); qemu_log_mask_and_addr(CPU_LOG_EXEC, itb->pc, "Trace %p [%d: " TARGET_FMT_lx "] %s\n", -@@ -365,6 +382,7 @@ +@@ -337,7 +352,7 @@ + TranslationBlock *tb; + target_ulong cs_base, pc; + uint32_t flags; +- bool have_tb_lock = false; ++ bool have_tb_lock = false, translated = false, chained = false; + + /* we record a subset of the CPU state. It will + always be the same before a given translated block +@@ -365,6 +380,7 @@ if (!tb) { /* if no translated code available, then translate it now */ tb = tb_gen_code(cpu, pc, cs_base, flags, 0); -+ afl_request_tsl(pc, cs_base, flags); ++ translated = true; } mmap_unlock(); +@@ -390,11 +406,15 @@ + } + if (!tb->invalid) { + tb_add_jump(last_tb, tb_exit, tb); ++ chained = true; + } + } + if (have_tb_lock) { + tb_unlock(); + } ++ if (translated || chained) { ++ afl_request_tsl(pc, cs_base, flags, chained ? last_tb : NULL, tb_exit); ++ } + return tb; + } + diff --git a/Instrumentor/patches-coverage/eclipser.c b/Instrumentor/patches-coverage/eclipser.c index 5b4d28c..2f5bf84 100644 --- a/Instrumentor/patches-coverage/eclipser.c +++ b/Instrumentor/patches-coverage/eclipser.c @@ -23,7 +23,7 @@ void eclipser_setup_before_forkserver(void); void eclipser_setup_after_forkserver(void); void eclipser_detach(void); void eclipser_exit(void); -void eclipser_log_bb(abi_ulong addr); +void helper_eclipser_log_bb(abi_ulong addr); abi_ulong eclipser_entry_point; /* ELF entry point (_start) */ @@ -112,7 +112,7 @@ void eclipser_exit(void) { } } -void eclipser_log_bb(abi_ulong addr) { +void helper_eclipser_log_bb(abi_ulong addr) { abi_ulong prev_addr_local; abi_ulong edge, hash; unsigned int byte_idx, byte_mask; diff --git a/Instrumentor/patches-coverage/makefile-objs.diff b/Instrumentor/patches-coverage/makefile-objs.diff index 5d5ee39..5f2f2d2 100644 --- a/Instrumentor/patches-coverage/makefile-objs.diff +++ b/Instrumentor/patches-coverage/makefile-objs.diff @@ -1,5 +1,5 @@ ---- qemu-2.10.0-coverage/accel/tcg/Makefile.objs.orig 2020-10-03 21:05:29.921244848 -0700 -+++ qemu-2.10.0-coverage/accel/tcg/Makefile.objs 2020-10-03 21:05:21.853199823 -0700 +--- qemu-2.10.0-coverage/accel/tcg/Makefile.objs.orig 2020-10-12 02:23:45.421904292 -0700 ++++ qemu-2.10.0-coverage/accel/tcg/Makefile.objs 2020-10-12 02:23:41.501944668 -0700 @@ -1,3 +1,3 @@ obj-$(CONFIG_SOFTMMU) += tcg-all.o obj-$(CONFIG_SOFTMMU) += cputlb.o diff --git a/Instrumentor/patches-coverage/syscall.diff b/Instrumentor/patches-coverage/syscall.diff index 93c3930..25888f9 100644 --- a/Instrumentor/patches-coverage/syscall.diff +++ b/Instrumentor/patches-coverage/syscall.diff @@ -1,5 +1,5 @@ ---- qemu-2.10.0-coverage/linux-user/syscall.c.orig 2020-10-03 21:05:29.921244848 -0700 -+++ qemu-2.10.0-coverage/linux-user/syscall.c 2020-10-03 21:05:21.897200066 -0700 +--- qemu-2.10.0-coverage/linux-user/syscall.c.orig 2020-10-12 02:23:45.421904292 -0700 ++++ qemu-2.10.0-coverage/linux-user/syscall.c 2020-10-12 02:23:41.517944503 -0700 @@ -116,6 +116,10 @@ #include "qemu.h" diff --git a/Instrumentor/patches-coverage/target-helper.diff b/Instrumentor/patches-coverage/target-helper.diff new file mode 100644 index 0000000..76adf4d --- /dev/null +++ b/Instrumentor/patches-coverage/target-helper.diff @@ -0,0 +1,12 @@ +--- qemu-2.10.0-coverage/target/i386/helper.h.orig 2020-10-12 02:23:45.429904211 -0700 ++++ qemu-2.10.0-coverage/target/i386/helper.h 2020-10-12 02:23:41.473944955 -0700 +@@ -226,3 +226,9 @@ + DEF_HELPER_3(rclq, tl, env, tl, tl) + DEF_HELPER_3(rcrq, tl, env, tl, tl) + #endif ++ ++#ifdef TARGET_X86_64 ++DEF_HELPER_1(eclipser_log_bb, void, i64) ++#else ++DEF_HELPER_1(eclipser_log_bb, void, i32) ++#endif diff --git a/Instrumentor/patches-coverage/target-translate.diff b/Instrumentor/patches-coverage/target-translate.diff new file mode 100644 index 0000000..33259f8 --- /dev/null +++ b/Instrumentor/patches-coverage/target-translate.diff @@ -0,0 +1,16 @@ +--- qemu-2.10.0-coverage/target/i386/translate.c.orig 2020-10-12 02:23:45.429904211 -0700 ++++ qemu-2.10.0-coverage/target/i386/translate.c 2020-10-12 02:23:41.477944915 -0700 +@@ -8390,6 +8390,13 @@ + int num_insns; + int max_insns; + ++#ifdef TARGET_X86_64 ++ TCGv_i64 pc_var = tcg_const_i64((uint64_t)tb->pc); ++#else ++ TCGv_i32 pc_var = tcg_const_i32((uint64_t)tb->pc); ++#endif ++ gen_helper_eclipser_log_bb(pc_var); ++ + /* generate intermediate code */ + pc_start = tb->pc; + cs_base = tb->cs_base; diff --git a/Instrumentor/prepare_qemu.sh b/Instrumentor/prepare_qemu.sh index 902d50f..2c8364f 100755 --- a/Instrumentor/prepare_qemu.sh +++ b/Instrumentor/prepare_qemu.sh @@ -132,6 +132,8 @@ cp patches-coverage/eclipser.c qemu-${VERSION}-coverage/accel/tcg/ patch -p0 Date: Mon, 12 Oct 2020 03:27:47 -0700 Subject: [PATCH 19/29] Cleanup target program executor engine Remove the code for --usepty option, since we no longer target coreutils programs. --- src/Core/Executor.fs | 14 ++++----- src/Core/Options.fs | 4 --- src/Core/libexec.c | 70 ++++++++------------------------------------ 3 files changed, 18 insertions(+), 70 deletions(-) diff --git a/src/Core/Executor.fs b/src/Core/Executor.fs index 2c2eec6..bef6bea 100644 --- a/src/Core/Executor.fs +++ b/src/Core/Executor.fs @@ -16,7 +16,7 @@ type Tracer = Coverage | Branch | BBCount [] extern int init_forkserver_coverage (int argc, string[] argv, uint64 timeout) [] extern int init_forkserver_branch (int argc, string[] argv, uint64 timeout) [] extern void kill_forkserver () -[] extern Signal exec (int argc, string[] argv, int stdin_size, byte[] stdin_data, uint64 timeout, bool use_pty) +[] extern Signal exec (int argc, string[] argv, int stdin_size, byte[] stdin_data, uint64 timeout) [] extern Signal exec_fork_coverage (uint64 timeout, int stdin_size, byte[] stdin_data) [] extern Signal exec_fork_branch (uint64 timeout, int stdin_size, byte[] stdin_data, uint64 targ_addr, uint32 targ_index, int measure_cov) @@ -106,12 +106,12 @@ let private setupFile seed = let private prepareStdIn seed = match seed.Source with | StdInput -> Seed.concretize seed - | FileInput filePath -> [| |] + | FileInput _ -> [| |] (*** Tracer result parsing functions ***) -/// TODO. Currently we only support edge coverage gain. Will extend the system -/// to support path coverage gain if needed. +// TODO. Currently we only support edge coverage gain. Will extend the system +// to support path coverage gain if needed. let private parseCoverage filename = match readAllLines filename with | [newEdgeFlag; _] -> if int newEdgeFlag = 1 then NewEdge else NoGain @@ -172,12 +172,11 @@ let private tryReadBranchInfo opt filename tryVal = let private runTracer tracerType opt (stdin: byte array) = let targetProg = opt.TargetProg let timeout = opt.ExecTimeout - let usePty = opt.UsePty let tracer = selectTracer tracerType opt.Architecture let cmdLine = opt.Arg.Split(WHITES, StringSplitOptions.RemoveEmptyEntries) let args = Array.append [|tracer; targetProg|] cmdLine let argc = args.Length - exec(argc, args, stdin.Length, stdin, timeout, usePty) + exec(argc, args, stdin.Length, stdin, timeout) let private runCoverageTracerForked opt stdin = let timeout = opt.ExecTimeout @@ -249,9 +248,8 @@ let nativeExecute opt seed = setupFile seed let stdin = prepareStdIn seed let timeout = opt.ExecTimeout - let usePty = opt.UsePty let cmdLine = opt.Arg.Split(WHITES, StringSplitOptions.RemoveEmptyEntries) let args = Array.append [| targetProg |] cmdLine let argc = args.Length - exec(argc, args, stdin.Length, stdin, timeout, usePty) + exec(argc, args, stdin.Length, stdin, timeout) diff --git a/src/Core/Options.fs b/src/Core/Options.fs index 325d828..78b3dac 100644 --- a/src/Core/Options.fs +++ b/src/Core/Options.fs @@ -12,7 +12,6 @@ type FuzzerCLI = // Options related to program execution. | [] [] [] Program of path: string | [] ExecTimeout of millisec:uint64 - | [] UsePty | [] Architecture of string // Options related to seed. | [] [] InputDir of path: string @@ -32,7 +31,6 @@ with // Options related to program execution. | Program _ -> "Target program for test case generation with fuzzing." | ExecTimeout _ -> "Execution timeout (ms) for a fuzz run (default:500)" - | UsePty _ -> "Use pseudo tty for standard input" | Architecture _ -> "Target program architecture (X86|X64) (default:X64)" // Options related to seed. | InputDir _ -> "Directory containing initial seeds." @@ -53,7 +51,6 @@ type FuzzOption = { // Options related to program execution. TargetProg : string ExecTimeout : uint64 - UsePty : bool Architecture : Arch // Options related to seed. InputDir : string @@ -76,7 +73,6 @@ let parseFuzzOption (args: string array) = // Options related to program execution. TargetProg = System.IO.Path.GetFullPath(r.GetResult (<@ Program @>)) ExecTimeout = r.GetResult (<@ ExecTimeout @>, defaultValue = DEF_EXEC_TO) - UsePty = r.Contains (<@ UsePty @>) Architecture = r.GetResult(<@ Architecture @>, defaultValue = "X64") |> Arch.ofString // Options related to seed. diff --git a/src/Core/libexec.c b/src/Core/libexec.c index cf8ba95..c4537a4 100755 --- a/src/Core/libexec.c +++ b/src/Core/libexec.c @@ -22,7 +22,6 @@ #include #include #include -#include #include #include #include @@ -38,15 +37,11 @@ static pid_t branch_forksrv_pid; static int branch_fsrv_ctl_fd, branch_fsrv_st_fd; static pid_t child_pid = 0; -static int pty_fd; -static int pty_flag; static int timeout_flag; static int non_fork_stdin_fd; static int coverage_stdin_fd; static int branch_stdin_fd; -pid_t (*sym_forkpty)(int *, char *, const struct termios*, const struct winsize *); - void error_exit(char* msg) { perror(msg); exit(-1); @@ -111,8 +106,6 @@ void initialize_exec(void) { struct sigaction sa; void* handle = dlopen("libutil.so.1", RTLD_LAZY); - sym_forkpty = (pid_t (*)(int *, char*, const struct termios*, const struct winsize*))dlsym(handle, "forkpty"); - sa.sa_flags = SA_RESTART; sa.sa_sigaction = NULL; @@ -135,8 +128,6 @@ int waitchild(pid_t pid, uint64_t timeout) perror("[Warning] waitpid() : "); alarm(0); // Cancle pending alarm - if (pty_flag) - close(pty_fd); if ( WIFEXITED( childstatus ) ) return 0; @@ -152,70 +143,34 @@ int waitchild(pid_t pid, uint64_t timeout) } } -void nonblocking_stdin() { - int flags; - if ((flags = fcntl(0, F_GETFL, 0)) == -1) - flags = 0; - if (fcntl(0, F_SETFL, flags | O_NONBLOCK) == -1) { - perror("fcntl"); - return; - } -} - -void term_setting(int pty_fd) { - struct termios newtio; - - tcgetattr(pty_fd, &newtio); - newtio.c_lflag &= ~ICANON & ~ECHO; - newtio.c_cc[VINTR] = 0; /* Ctrl-c */ - newtio.c_cc[VQUIT] = 0; /* Ctrl-\ */ - newtio.c_cc[VSUSP] = 0; /* Ctrl-z */ - tcsetattr(pty_fd, TCSADRAIN, &newtio); -} - -int exec(int argc, char **args, int stdin_size, char *stdin_data, uint64_t timeout, int use_pty) { +int exec(int argc, char **args, int stdin_size, char *stdin_data, uint64_t timeout) { int i, devnull, ret; - char **argv = (char **)malloc( sizeof(char*) * (argc + 1) ); + char **argv = (char **)malloc(sizeof(char*) * (argc + 1)); if (!argv) error_exit( "args malloc" ); - for (i = 0; i 0) { - if (pty_flag) { - term_setting(pty_fd); - if(write(pty_fd, stdin_data, stdin_size) != stdin_size) - error_exit("exec() : write(pty_fd, ...)"); - } - free(argv); } else { error_exit("fork"); @@ -370,7 +325,6 @@ int exec_fork_coverage(uint64_t timeout, int stdin_size, char *stdin_data) { static struct itimerval it; static unsigned char tmp[4]; - /* TODO : what if we want to use pseudo-terminal? */ write_stdin(coverage_stdin_fd, stdin_size, stdin_data); if ((res = write(coverage_fsrv_ctl_fd, tmp, 4)) != 4) { From 8aebe10c8274f1a7e2f86439bb8f4109a330cbc7 Mon Sep 17 00:00:00 2001 From: Jaeseung Choi Date: Mon, 12 Oct 2020 03:53:34 -0700 Subject: [PATCH 20/29] Introduce --noforkserver option --- examples/test_no_fork_server.sh | 10 ++++++++++ src/Core/Executor.fs | 34 ++++++++++++++++++--------------- src/Core/Options.fs | 4 ++++ 3 files changed, 33 insertions(+), 15 deletions(-) create mode 100755 examples/test_no_fork_server.sh diff --git a/examples/test_no_fork_server.sh b/examples/test_no_fork_server.sh new file mode 100755 index 0000000..e75f5e9 --- /dev/null +++ b/examples/test_no_fork_server.sh @@ -0,0 +1,10 @@ +#!/bin/bash + +# Grey-box concolic should find more than 9 file input test cases that have node +# coverage gain. +gcc cmp.c -o cmp.bin -static -g || exit 1 +rm -rf box +mkdir box +cd box +dotnet ../../build/Eclipser.dll \ + -p ../cmp.bin -t 5 -v 1 -o output -f input --arg input --noforkserver diff --git a/src/Core/Executor.fs b/src/Core/Executor.fs index bef6bea..3abbce6 100644 --- a/src/Core/Executor.fs +++ b/src/Core/Executor.fs @@ -47,6 +47,20 @@ let mutable bitmapLog = "" let mutable dbgLog = "" let mutable forkServerEnabled = false +let private initializeForkServer opt = + forkServerEnabled <- true + let cmdLine = opt.Arg.Split(WHITES, StringSplitOptions.RemoveEmptyEntries) + let coverageTracer = selectTracer Coverage opt.Architecture + let args = Array.append [|coverageTracer; opt.TargetProg|] cmdLine + let pidCoverage = init_forkserver_coverage(args.Length, args, opt.ExecTimeout) + if pidCoverage = -1 then + failwith "Failed to initialize fork server for coverage tracer" + let branchTracer = selectTracer Branch opt.Architecture + let args = Array.append [|branchTracer; opt.TargetProg|] cmdLine + let pidBranch = init_forkserver_branch (args.Length, args, opt.ExecTimeout) + if pidBranch = -1 then + failwith "Failed to initialize fork server for branch tracer" + let initialize opt = let outDir = opt.OutDir let verbosity = opt.Verbosity @@ -62,22 +76,12 @@ let initialize opt = set_env("ECL_BITMAP_LOG", System.IO.Path.GetFullPath(bitmapLog)) if verbosity >= 2 then set_env("ECL_DBG_LOG", System.IO.Path.GetFullPath(dbgLog)) - // Initialize C wrapper code. initialize_exec () - // Initialize fork server. - forkServerEnabled <- true - set_env("ECL_FORK_SERVER", "1") - let cmdLine = opt.Arg.Split(WHITES, StringSplitOptions.RemoveEmptyEntries) - let coverageTracer = selectTracer Coverage opt.Architecture - let args = Array.append [|coverageTracer; opt.TargetProg|] cmdLine - let pidCoverage = init_forkserver_coverage(args.Length, args, opt.ExecTimeout) - if pidCoverage = -1 then - failwith "Failed to initialize fork server for coverage tracer" - let branchTracer = selectTracer Branch opt.Architecture - let args = Array.append [|branchTracer; opt.TargetProg|] cmdLine - let pidBranch = init_forkserver_branch (args.Length, args, opt.ExecTimeout) - if pidBranch = -1 then - failwith "Failed to initialize fork server for branch tracer" + if opt.ForkServer then + set_env("ECL_FORK_SERVER", "1") + initializeForkServer opt + else + set_env("ECL_FORK_SERVER", "0") let cleanup () = if forkServerEnabled then kill_forkserver () diff --git a/src/Core/Options.fs b/src/Core/Options.fs index 78b3dac..b1134b8 100644 --- a/src/Core/Options.fs +++ b/src/Core/Options.fs @@ -13,6 +13,7 @@ type FuzzerCLI = | [] [] [] Program of path: string | [] ExecTimeout of millisec:uint64 | [] Architecture of string + | [] NoForkServer // Options related to seed. | [] [] InputDir of path: string | [] Arg of string @@ -32,6 +33,7 @@ with | Program _ -> "Target program for test case generation with fuzzing." | ExecTimeout _ -> "Execution timeout (ms) for a fuzz run (default:500)" | Architecture _ -> "Target program architecture (X86|X64) (default:X64)" + | NoForkServer -> "Do not use fork server for target program execution" // Options related to seed. | InputDir _ -> "Directory containing initial seeds." | Arg _ -> "Command-line argument of program under test." @@ -52,6 +54,7 @@ type FuzzOption = { TargetProg : string ExecTimeout : uint64 Architecture : Arch + ForkServer : bool // Options related to seed. InputDir : string Arg : string @@ -75,6 +78,7 @@ let parseFuzzOption (args: string array) = ExecTimeout = r.GetResult (<@ ExecTimeout @>, defaultValue = DEF_EXEC_TO) Architecture = r.GetResult(<@ Architecture @>, defaultValue = "X64") |> Arch.ofString + ForkServer = not (r.Contains(<@ NoForkServer @>)) // Enable by default. // Options related to seed. InputDir = r.GetResult(<@ InputDir @>, defaultValue = "") Arg = r.GetResult (<@ Arg @>, defaultValue = "") From ee7c75c487f2168c24426e73e04448be2561200a Mon Sep 17 00:00:00 2001 From: Jaeseung Choi Date: Mon, 12 Oct 2020 03:53:50 -0700 Subject: [PATCH 21/29] Replace hard-coded bitmap size into a constant macro --- Instrumentor/patches-branch/eclipser.c | 10 ++++++---- Instrumentor/patches-coverage/eclipser.c | 11 +++++++---- src/Core/Executor.fs | 4 +++- 3 files changed, 16 insertions(+), 9 deletions(-) diff --git a/Instrumentor/patches-branch/eclipser.c b/Instrumentor/patches-branch/eclipser.c index 27b6b59..825afeb 100644 --- a/Instrumentor/patches-branch/eclipser.c +++ b/Instrumentor/patches-branch/eclipser.c @@ -22,6 +22,8 @@ extern unsigned int afl_forksrv_pid; #define FORKSRV_FD 198 #define TSL_FD (FORKSRV_FD - 1) +#define BITMAP_SIZE (0x10000) +#define BITMAP_MASK (BITMAP_SIZE - 1) #define MAX_TRACE_LEN (100000) #define IGNORE_COVERAGE 1 @@ -66,7 +68,7 @@ void flush_trace_buffer(void) { void eclipser_setup_before_forkserver(void) { char * bitmap_path = getenv("ECL_BITMAP_LOG"); int bitmap_fd = open(bitmap_path, O_RDWR | O_CREAT, 0644); - edge_bitmap = (unsigned char*) mmap(NULL, 0x10000, PROT_READ | PROT_WRITE, MAP_SHARED, bitmap_fd, 0); + edge_bitmap = (unsigned char*) mmap(NULL, BITMAP_SIZE, PROT_READ | PROT_WRITE, MAP_SHARED, bitmap_fd, 0); assert(edge_bitmap != (void *) -1); coverage_path = getenv("ECL_COVERAGE_LOG"); @@ -112,7 +114,7 @@ void eclipser_detach(void) { close(TSL_FD); if (edge_bitmap) { - munmap(edge_bitmap, 0x10000); + munmap(edge_bitmap, BITMAP_SIZE); edge_bitmap = NULL; } } @@ -141,7 +143,7 @@ void eclipser_exit(void) { } if (edge_bitmap) { - munmap(edge_bitmap, 0x10000); + munmap(edge_bitmap, BITMAP_SIZE); edge_bitmap = NULL; } } @@ -329,7 +331,7 @@ void helper_eclipser_log_bb(abi_ulong addr) { // Update bitmap. hash = (edge >> 4) ^ (edge << 8); - byte_idx = (hash >> 3) & 0xffff; + byte_idx = (hash >> 3) & BITMAP_MASK; byte_mask = 1 << (hash & 0x7); // Use the lowest 3 bits to shift old_byte = edge_bitmap[byte_idx]; new_byte = old_byte | byte_mask; diff --git a/Instrumentor/patches-coverage/eclipser.c b/Instrumentor/patches-coverage/eclipser.c index 2f5bf84..a22abc5 100644 --- a/Instrumentor/patches-coverage/eclipser.c +++ b/Instrumentor/patches-coverage/eclipser.c @@ -19,6 +19,9 @@ extern unsigned int afl_forksrv_pid; #define FORKSRV_FD 198 #define TSL_FD (FORKSRV_FD - 1) +#define BITMAP_SIZE (0x10000) +#define BITMAP_MASK (BITMAP_SIZE - 1) + void eclipser_setup_before_forkserver(void); void eclipser_setup_after_forkserver(void); void eclipser_detach(void); @@ -40,7 +43,7 @@ static unsigned char * edge_bitmap = NULL; void eclipser_setup_before_forkserver(void) { char * bitmap_path = getenv("ECL_BITMAP_LOG"); int bitmap_fd = open(bitmap_path, O_RDWR | O_CREAT, 0644); - edge_bitmap = (unsigned char*) mmap(NULL, 0x10000, PROT_READ | PROT_WRITE, MAP_SHARED, bitmap_fd, 0); + edge_bitmap = (unsigned char*) mmap(NULL, BITMAP_SIZE, PROT_READ | PROT_WRITE, MAP_SHARED, bitmap_fd, 0); assert(edge_bitmap != (void *) -1); coverage_path = getenv("ECL_COVERAGE_LOG"); @@ -78,7 +81,7 @@ void eclipser_detach(void) { } if (edge_bitmap) { - munmap(edge_bitmap, 0x10000); + munmap(edge_bitmap, BITMAP_SIZE); edge_bitmap = NULL; } @@ -107,7 +110,7 @@ void eclipser_exit(void) { } if (edge_bitmap) { - munmap(edge_bitmap, 0x10000); + munmap(edge_bitmap, BITMAP_SIZE); edge_bitmap = NULL; } } @@ -133,7 +136,7 @@ void helper_eclipser_log_bb(abi_ulong addr) { // Update bitmap. hash = (edge >> 4) ^ (edge << 8); - byte_idx = (hash >> 3) & 0xffff; + byte_idx = (hash >> 3) & BITMAP_MASK; byte_mask = 1 << (hash & 0x7); // Use the lowest 3 bits to shift old_byte = edge_bitmap[byte_idx]; new_byte = old_byte | byte_mask; diff --git a/src/Core/Executor.fs b/src/Core/Executor.fs index 3abbce6..6494bef 100644 --- a/src/Core/Executor.fs +++ b/src/Core/Executor.fs @@ -6,6 +6,8 @@ open System.Runtime.InteropServices open Utils open Options +let private BITMAP_SIZE = 0x10000L + let private WHITES = [| ' '; '\t'; '\n' |] /// Kinds of QEMU instrumentor. Each instrumentor serves different purposes. @@ -72,7 +74,7 @@ let initialize opt = set_env("ECL_BRANCH_LOG", System.IO.Path.GetFullPath(branchLog)) set_env("ECL_COVERAGE_LOG", System.IO.Path.GetFullPath(coverageLog)) use bitmapFile = File.Create(bitmapLog) - bitmapFile.SetLength(0x10000L) + bitmapFile.SetLength(BITMAP_SIZE) set_env("ECL_BITMAP_LOG", System.IO.Path.GetFullPath(bitmapLog)) if verbosity >= 2 then set_env("ECL_DBG_LOG", System.IO.Path.GetFullPath(dbgLog)) From ba2140fa1a2adeb39674f2c0857f11154a069576 Mon Sep 17 00:00:00 2001 From: Jaeseung Choi Date: Tue, 13 Oct 2020 14:41:35 +0900 Subject: [PATCH 22/29] Reintroduce a resource scheduler Now we rely on AFL for random-based fuzzing, so we should indirectly control the resource use by making Eclipser to yield some of the system resource. --- src/Core/Executor.fs | 59 +++++++++++++++++++++++++------------------ src/Eclipser.fsproj | 1 + src/Fuzz/Fuzz.fs | 27 ++++++++++++-------- src/Fuzz/Scheduler.fs | 45 +++++++++++++++++++++++++++++++++ src/Fuzz/TestCase.fs | 56 ++++++++++++++++++++++------------------ 5 files changed, 127 insertions(+), 61 deletions(-) create mode 100644 src/Fuzz/Scheduler.fs diff --git a/src/Core/Executor.fs b/src/Core/Executor.fs index 6494bef..bb4d7fb 100644 --- a/src/Core/Executor.fs +++ b/src/Core/Executor.fs @@ -22,6 +22,13 @@ type Tracer = Coverage | Branch | BBCount [] extern Signal exec_fork_coverage (uint64 timeout, int stdin_size, byte[] stdin_data) [] extern Signal exec_fork_branch (uint64 timeout, int stdin_size, byte[] stdin_data, uint64 targ_addr, uint32 targ_index, int measure_cov) +let mutable private branchLog = "" +let mutable private coverageLog = "" +let mutable private bitmapLog = "" +let mutable private dbgLog = "" +let mutable private forkServerOn = false +let mutable private roundExecutions = 0 + (*** Tracer and file paths ***) let buildDir = @@ -43,14 +50,10 @@ let selectTracer tracer arch = | BBCount, X86 -> bbCountTracerX86 | BBCount, X64 -> bbCountTracerX64 -let mutable branchLog = "" -let mutable coverageLog = "" -let mutable bitmapLog = "" -let mutable dbgLog = "" -let mutable forkServerEnabled = false +(*** Initialization and cleanup ***) let private initializeForkServer opt = - forkServerEnabled <- true + forkServerOn <- true let cmdLine = opt.Arg.Split(WHITES, StringSplitOptions.RemoveEmptyEntries) let coverageTracer = selectTracer Coverage opt.Architecture let args = Array.append [|coverageTracer; opt.TargetProg|] cmdLine @@ -85,18 +88,24 @@ let initialize opt = else set_env("ECL_FORK_SERVER", "0") +let abandonForkServer () = + log "Abandon fork server" + forkServerOn <- false + set_env("ECL_FORK_SERVER", "0") + kill_forkserver () + let cleanup () = - if forkServerEnabled then kill_forkserver () + if forkServerOn then kill_forkserver () removeFile branchLog removeFile coverageLog removeFile bitmapLog removeFile dbgLog -let abandonForkServer () = - log "Abandon fork server" - forkServerEnabled <- false - set_env("ECL_FORK_SERVER", "0") - kill_forkserver () +(*** Execution count statistics ***) + +let getRoundExecutions () = roundExecutions + +let resetRoundExecutions () = roundExecutions <- 0 (*** File handling utilities ***) @@ -173,9 +182,10 @@ let private tryReadBranchInfo opt filename tryVal = | [ branchInfo ] -> Some branchInfo | _ -> None -(*** Tracer execute functions ***) +(*** Tracer execution functions ***) let private runTracer tracerType opt (stdin: byte array) = + roundExecutions <- roundExecutions + 1 let targetProg = opt.TargetProg let timeout = opt.ExecTimeout let tracer = selectTracer tracerType opt.Architecture @@ -185,6 +195,7 @@ let private runTracer tracerType opt (stdin: byte array) = exec(argc, args, stdin.Length, stdin, timeout) let private runCoverageTracerForked opt stdin = + roundExecutions <- roundExecutions + 1 let timeout = opt.ExecTimeout let stdLen = Array.length stdin let signal = exec_fork_coverage(timeout, stdLen, stdin) @@ -192,6 +203,7 @@ let private runCoverageTracerForked opt stdin = signal let private runBranchTracerForked opt stdin addr idx covMeasure = + roundExecutions <- roundExecutions + 1 let timeout = opt.ExecTimeout let stdLen = Array.length stdin let covEnum = CoverageMeasure.toEnum covMeasure @@ -204,12 +216,12 @@ let private setEnvForBranch (addr: uint64) (idx: uint32) covMeasure = set_env("ECL_BRANCH_IDX", sprintf "%016x" idx) set_env("ECL_MEASURE_COV", sprintf "%d" (CoverageMeasure.toEnum covMeasure)) -(*** Top-level tracer executor functions ***) +(*** Top-level tracer execution functions ***) let getCoverage opt seed = setupFile seed let stdin = prepareStdIn seed - let exitSig = if forkServerEnabled then runCoverageTracerForked opt stdin + let exitSig = if forkServerOn then runCoverageTracerForked opt stdin else runTracer Coverage opt stdin let coverageGain = parseCoverage coverageLog (exitSig, coverageGain) @@ -217,10 +229,9 @@ let getCoverage opt seed = let getBranchTrace opt seed tryVal = setupFile seed let stdin = prepareStdIn seed - let exitSig = if forkServerEnabled - then runBranchTracerForked opt stdin 0UL 0ul NonCumulative - else setEnvForBranch 0UL 0ul NonCumulative - runTracer Branch opt stdin + let exitSig = + if forkServerOn then runBranchTracerForked opt stdin 0UL 0ul NonCumulative + else setEnvForBranch 0UL 0ul NonCumulative; runTracer Branch opt stdin let coverageGain = parseCoverage coverageLog let branchTrace = readBranchTrace opt branchLog tryVal removeFile coverageLog @@ -230,10 +241,9 @@ let getBranchInfo opt seed tryVal targPoint = setupFile seed let stdin = prepareStdIn seed let addr, idx = targPoint.Addr, uint32 targPoint.Idx - let exitSig = if forkServerEnabled - then runBranchTracerForked opt stdin addr idx Cumulative - else setEnvForBranch addr idx Cumulative - runTracer Branch opt stdin + let exitSig = + if forkServerOn then runBranchTracerForked opt stdin addr idx Cumulative + else setEnvForBranch addr idx Cumulative; runTracer Branch opt stdin let coverageGain = parseCoverage coverageLog let branchInfoOpt = tryReadBranchInfo opt branchLog tryVal removeFile coverageLog @@ -243,7 +253,7 @@ let getBranchInfoOnly opt seed tryVal targPoint = setupFile seed let stdin = prepareStdIn seed let addr, idx = targPoint.Addr, uint32 targPoint.Idx - if forkServerEnabled then runBranchTracerForked opt stdin addr idx Ignore + if forkServerOn then runBranchTracerForked opt stdin addr idx Ignore else setEnvForBranch addr idx Ignore; runTracer Branch opt stdin |> ignore let brInfoOpt = tryReadBranchInfo opt branchLog tryVal @@ -258,4 +268,3 @@ let nativeExecute opt seed = let args = Array.append [| targetProg |] cmdLine let argc = args.Length exec(argc, args, stdin.Length, stdin, timeout) - diff --git a/src/Eclipser.fsproj b/src/Eclipser.fsproj index 300a8dd..1a6f820 100644 --- a/src/Eclipser.fsproj +++ b/src/Eclipser.fsproj @@ -26,6 +26,7 @@ + diff --git a/src/Fuzz/Fuzz.fs b/src/Fuzz/Fuzz.fs index 04686f8..4106482 100644 --- a/src/Fuzz/Fuzz.fs +++ b/src/Fuzz/Fuzz.fs @@ -4,11 +4,11 @@ open System.Threading open Utils open Options +// Synchronize the seed queue with AFL every SYNC_N iterations. +let SYNC_N = 10 + let private printFoundSeed verbosity seed = - if verbosity >= 1 then - log "[*] Found by grey-box concolic: %s" (Seed.toString seed) - elif verbosity >= 0 then - log "[*] Found by grey-box concolic %s" (Seed.toString seed) + if verbosity >= 1 then log "[*] Found a new seed: %s" (Seed.toString seed) let private evalSeed opt seed exitSig covGain = TestCase.save opt seed exitSig covGain @@ -40,21 +40,25 @@ let private makeSteppedItems pr seed = | None -> [] | Some s -> [(pr, s)] -let syncWithAFL opt seedQueue n = - // Sychronize the seed queue with AFL instances every ten iterations. - if n % 10 = 0 && opt.SyncDir <> "" then Sync.run opt seedQueue +// Decides how to share the resource with AFL instances. +let private scheduleWithAFL opt = + if opt.SyncDir <> "" then Scheduler.checkAndReserveTime () + +// Sychronize the seed queue with AFL instances. +let private syncWithAFL opt seedQueue n = + if opt.SyncDir <> "" && n % SYNC_N = 0 then Sync.run opt seedQueue else seedQueue let rec private fuzzLoop opt seedQueue n = - let verbosity = opt.Verbosity + scheduleWithAFL opt let seedQueue = syncWithAFL opt seedQueue n if SeedQueue.isEmpty seedQueue then - if n % 10 = 0 && verbosity >= 1 then log "Seed queue empty, waiting..." + if n % 10 = 0 && opt.Verbosity >= 1 then log "Seed queue empty, waiting..." Thread.Sleep(1000) fuzzLoop opt seedQueue (n + 1) else let priority, seed, seedQueue = SeedQueue.dequeue seedQueue - if verbosity >= 1 then log "Fuzzing with: %s" (Seed.toString seed) + if opt.Verbosity >= 1 then log "Fuzzing with: %s" (Seed.toString seed) let newItems = GreyConcolic.run seed opt // Relocate the cursors of newly generated seeds. let relocatedItems = makeRelocatedItems opt newItems @@ -86,12 +90,13 @@ let main args = createDirectoryIfNotExists opt.OutDir TestCase.initialize opt.OutDir Executor.initialize opt + Scheduler.initialize () let emptyQueue = SeedQueue.initialize () let initialSeeds = initializeSeeds opt log "[*] Total %d initial seeds" (List.length initialSeeds) let initItems = List.choose (makeInitialItems opt) initialSeeds let initQueue = List.fold SeedQueue.enqueue emptyQueue initItems - log "[*] Start fuzzing" Async.Start (fuzzingTimer opt.Timelimit) + log "[*] Start fuzzing" fuzzLoop opt initQueue 0 0 // Unreachable diff --git a/src/Fuzz/Scheduler.fs b/src/Fuzz/Scheduler.fs new file mode 100644 index 0000000..e1c4129 --- /dev/null +++ b/src/Fuzz/Scheduler.fs @@ -0,0 +1,45 @@ +module Eclipser.Scheduler + +open Utils +open System.Threading + +let private timer = new System.Diagnostics.Stopwatch() +// We will consider every ROUND_SIZE executions as a single round. +let private ROUND_SIZE = 10000 +// Tentative efficiency of random fuzzing. TODO: Communicate with AFL for this. +let private RAND_FUZZ_EFFICIENCY = 0.0005 +let private SLEEP_FACTOR_MIN = 0.0 +let private SLEEP_FACTOR_MAX = 4.0 + +// Decides sleep factor 'f', which will be used to sleep for 'f * elapsed time'. +// This means we will utilize 1 / (2 * (f + 1)) of the system resource. +let private decideSleepFactor roundExecs roundTCs = + let greyConcEfficiency = float (roundTCs) / float (roundExecs) + // GREY_CONC_EFF : RAND_FUZZ_EFF = 1 : 2 * factor + 1 + let factor = if greyConcEfficiency = 0.0 then SLEEP_FACTOR_MAX + else (RAND_FUZZ_EFFICIENCY / greyConcEfficiency - 1.0) / 2.0 + log "[*] Grey-concolic eff. = %.3f, factor = %.3f" greyConcEfficiency factor + // Bound the factor between minimum and maximum value allowed. + max SLEEP_FACTOR_MIN (min SLEEP_FACTOR_MAX factor) + +let initialize () = + Executor.resetRoundExecutions() + TestCase.resetRoundTestCaseCount() + timer.Start() + +// Check the efficiency of the system and sleep for a while to adjust the weight +// of resource use with AFL. +let checkAndReserveTime () = + let roundExecs = Executor.getRoundExecutions () + if roundExecs > ROUND_SIZE then + let roundTCs = TestCase.getRoundTestCaseCount () + Executor.resetRoundExecutions() + TestCase.resetRoundTestCaseCount() + let sleepFactor = decideSleepFactor roundExecs roundTCs + let roundElapsed = timer.ElapsedMilliseconds + log "[*] Elapsed round time: %d sec." (roundElapsed / 1000L) + timer.Reset() + let sleepTime = int (float (roundElapsed) * sleepFactor) + log "[*] Decided sleep time: %d sec." (sleepTime / 1000) + Thread.Sleep(sleepTime) + timer.Start() diff --git a/src/Fuzz/TestCase.fs b/src/Fuzz/TestCase.fs index d8ba27f..88c710c 100644 --- a/src/Fuzz/TestCase.fs +++ b/src/Fuzz/TestCase.fs @@ -16,47 +16,53 @@ let initialize outDir = (*** Statistics ***) -let mutable private segfaultCount = 0 -let mutable private illegalInstrCount = 0 -let mutable private fpErrorCount = 0 -let mutable private abortCount = 0 -let mutable private crashCount = 0 -let mutable private testCaseCount = 0 +let mutable private totalSegfaults = 0 +let mutable private totalIllegals = 0 +let mutable private totalFPEs = 0 +let mutable private totalAborts = 0 +let mutable private totalCrashes = 0 +let mutable private totalTestCases = 0 +let mutable private roundTestCases = 0 let printStatistics () = - log "Testcases : %d" testCaseCount - log "Crashes : %d" crashCount - log " Segfault : %d" segfaultCount - log " Illegal instruction : %d" illegalInstrCount - log " Floating point error : %d" fpErrorCount - log " Program abortion : %d" abortCount - -let private updateCrashCount exitSig = + log "Testcases : %d" totalTestCases + log "Crashes : %d" totalCrashes + log " Segfault : %d" totalSegfaults + log " Illegal instruction : %d" totalIllegals + log " Floating point error : %d" totalFPEs + log " Program abortion : %d" totalAborts + +let private incrCrashCount exitSig = match exitSig with - | Signal.SIGSEGV -> segfaultCount <- segfaultCount + 1 - | Signal.SIGILL -> illegalInstrCount <- illegalInstrCount + 1 - | Signal.SIGFPE -> fpErrorCount <- fpErrorCount + 1 - | Signal.SIGABRT -> abortCount <- abortCount + 1 + | Signal.SIGSEGV -> totalSegfaults <- totalSegfaults + 1 + | Signal.SIGILL -> totalIllegals <- totalIllegals + 1 + | Signal.SIGFPE -> totalFPEs <- totalFPEs + 1 + | Signal.SIGABRT -> totalAborts <- totalAborts + 1 | _ -> failwith "updateCrashCount() called with a non-crashing exit signal" - crashCount <- crashCount + 1 + totalCrashes <- totalCrashes + 1 + +let private incrTestCaseCount () = + totalTestCases <- totalTestCases + 1 + roundTestCases <- roundTestCases + 1 + +let getRoundTestCaseCount () = roundTestCases -let private updateTestcaseCount () = - testCaseCount <- testCaseCount + 1 +let resetRoundTestCaseCount () = roundTestCases <- 0 (*** Test case storing functions ***) let private dumpCrash opt seed exitSig = if opt.Verbosity >= 0 then log "Save crash seed : %s" (Seed.toString seed) - let crashName = sprintf "id:%06d" crashCount + let crashName = sprintf "id:%06d" totalCrashes let crashPath = System.IO.Path.Combine(crashDir, crashName) System.IO.File.WriteAllBytes(crashPath, Seed.concretize seed) - updateCrashCount exitSig + incrCrashCount exitSig let private dumpTestCase seed = - let tcName = sprintf "id:%06d" testCaseCount + let tcName = sprintf "id:%06d" totalTestCases let tcPath = System.IO.Path.Combine(testcaseDir, tcName) System.IO.File.WriteAllBytes(tcPath, Seed.concretize seed) - updateTestcaseCount () + incrTestCaseCount () let private checkCrash opt seed exitSig covGain = if Signal.isCrash exitSig && covGain = NewEdge then (true, exitSig) From 191dea5477a61d308f725d0bac24836c7d70a82a Mon Sep 17 00:00:00 2001 From: Jaeseung Choi Date: Wed, 14 Oct 2020 03:33:04 -0700 Subject: [PATCH 23/29] Gather configuration values in a single file --- src/Core/Config.fs | 13 +++++++++++++ src/Core/Executor.fs | 35 +++++++++++++++++++---------------- src/Fuzz/Fuzz.fs | 4 +--- src/Fuzz/Scheduler.fs | 6 ++---- 4 files changed, 35 insertions(+), 23 deletions(-) diff --git a/src/Core/Config.fs b/src/Core/Config.fs index 8146bc0..62c074c 100644 --- a/src/Core/Config.fs +++ b/src/Core/Config.fs @@ -1,5 +1,18 @@ module Eclipser.Config +/// Size of bitmap to measure edge coverage. Should be updated along with the +/// macros at Instrumentor/patches-*/eclipser.c +let BITMAP_SIZE = 0x10000L + +/// Synchronize the seed queue with AFL every SYNC_N iteration of fuzzing loop. +let SYNC_N = 10 + +/// We will consider every ROUND_SIZE executions as a single round. A 'round' is +/// the unit of time for resource scheduling (cf. Scheduler.fs) +let ROUND_SIZE = 10000 +let SLEEP_FACTOR_MIN = 0.0 +let SLEEP_FACTOR_MAX = 4.0 + /// Default execution timeout of target program. let DEF_EXEC_TO = 500UL diff --git a/src/Core/Executor.fs b/src/Core/Executor.fs index bb4d7fb..185ed8a 100644 --- a/src/Core/Executor.fs +++ b/src/Core/Executor.fs @@ -3,13 +3,10 @@ module Eclipser.Executor open System open System.IO open System.Runtime.InteropServices +open Config open Utils open Options -let private BITMAP_SIZE = 0x10000L - -let private WHITES = [| ' '; '\t'; '\n' |] - /// Kinds of QEMU instrumentor. Each instrumentor serves different purposes. type Tracer = Coverage | Branch | BBCount @@ -50,11 +47,21 @@ let selectTracer tracer arch = | BBCount, X86 -> bbCountTracerX86 | BBCount, X64 -> bbCountTracerX64 +(*** Misc utility functions ***) + +let splitCmdLineArg (argStr: string) = + let whiteSpaces = [| ' '; '\t'; '\n' |] + argStr.Split(whiteSpaces, StringSplitOptions.RemoveEmptyEntries) + +let readAllLines filename = + try List.ofSeq (System.IO.File.ReadLines filename) with + | :? System.IO.FileNotFoundException -> [] + (*** Initialization and cleanup ***) let private initializeForkServer opt = forkServerOn <- true - let cmdLine = opt.Arg.Split(WHITES, StringSplitOptions.RemoveEmptyEntries) + let cmdLine = splitCmdLineArg opt.Arg let coverageTracer = selectTracer Coverage opt.Architecture let args = Array.append [|coverageTracer; opt.TargetProg|] cmdLine let pidCoverage = init_forkserver_coverage(args.Length, args, opt.ExecTimeout) @@ -107,11 +114,12 @@ let getRoundExecutions () = roundExecutions let resetRoundExecutions () = roundExecutions <- 0 -(*** File handling utilities ***) +(*** Setup functions ***) -let readAllLines filename = - try List.ofSeq (System.IO.File.ReadLines filename) with - | :? System.IO.FileNotFoundException -> [] +let private setEnvForBranch (addr: uint64) (idx: uint32) covMeasure = + set_env("ECL_BRANCH_ADDR", sprintf "%016x" addr) + set_env("ECL_BRANCH_IDX", sprintf "%016x" idx) + set_env("ECL_MEASURE_COV", sprintf "%d" (CoverageMeasure.toEnum covMeasure)) let private setupFile seed = match seed.Source with @@ -189,7 +197,7 @@ let private runTracer tracerType opt (stdin: byte array) = let targetProg = opt.TargetProg let timeout = opt.ExecTimeout let tracer = selectTracer tracerType opt.Architecture - let cmdLine = opt.Arg.Split(WHITES, StringSplitOptions.RemoveEmptyEntries) + let cmdLine = splitCmdLineArg opt.Arg let args = Array.append [|tracer; targetProg|] cmdLine let argc = args.Length exec(argc, args, stdin.Length, stdin, timeout) @@ -211,11 +219,6 @@ let private runBranchTracerForked opt stdin addr idx covMeasure = if signal = Signal.ERROR then abandonForkServer () signal -let private setEnvForBranch (addr: uint64) (idx: uint32) covMeasure = - set_env("ECL_BRANCH_ADDR", sprintf "%016x" addr) - set_env("ECL_BRANCH_IDX", sprintf "%016x" idx) - set_env("ECL_MEASURE_COV", sprintf "%d" (CoverageMeasure.toEnum covMeasure)) - (*** Top-level tracer execution functions ***) let getCoverage opt seed = @@ -264,7 +267,7 @@ let nativeExecute opt seed = setupFile seed let stdin = prepareStdIn seed let timeout = opt.ExecTimeout - let cmdLine = opt.Arg.Split(WHITES, StringSplitOptions.RemoveEmptyEntries) + let cmdLine = splitCmdLineArg opt.Arg let args = Array.append [| targetProg |] cmdLine let argc = args.Length exec(argc, args, stdin.Length, stdin, timeout) diff --git a/src/Fuzz/Fuzz.fs b/src/Fuzz/Fuzz.fs index 4106482..7825a88 100644 --- a/src/Fuzz/Fuzz.fs +++ b/src/Fuzz/Fuzz.fs @@ -1,12 +1,10 @@ module Eclipser.Fuzz open System.Threading +open Config open Utils open Options -// Synchronize the seed queue with AFL every SYNC_N iterations. -let SYNC_N = 10 - let private printFoundSeed verbosity seed = if verbosity >= 1 then log "[*] Found a new seed: %s" (Seed.toString seed) diff --git a/src/Fuzz/Scheduler.fs b/src/Fuzz/Scheduler.fs index e1c4129..b7f0a13 100644 --- a/src/Fuzz/Scheduler.fs +++ b/src/Fuzz/Scheduler.fs @@ -1,15 +1,13 @@ module Eclipser.Scheduler +open Config open Utils open System.Threading let private timer = new System.Diagnostics.Stopwatch() -// We will consider every ROUND_SIZE executions as a single round. -let private ROUND_SIZE = 10000 + // Tentative efficiency of random fuzzing. TODO: Communicate with AFL for this. let private RAND_FUZZ_EFFICIENCY = 0.0005 -let private SLEEP_FACTOR_MIN = 0.0 -let private SLEEP_FACTOR_MAX = 4.0 // Decides sleep factor 'f', which will be used to sleep for 'f * elapsed time'. // This means we will utilize 1 / (2 * (f + 1)) of the system resource. From 3f100191efcb9324751c350ab3b690b4c54351ae Mon Sep 17 00:00:00 2001 From: Jaeseung Choi Date: Wed, 14 Oct 2020 03:33:22 -0700 Subject: [PATCH 24/29] Update test examples - Add an example to test standard input fuzzing. - Clean up and refactor examples and scripts. --- examples/{extend.c => length.c} | 3 +- examples/{cmp.c => linear.c} | 5 +-- examples/loop.c | 4 -- examples/monoton.c | 2 - examples/nested.c | 4 -- examples/stdin.c | 67 +++++++++++++++++++++++++++++++++ examples/test_32bit_clang.sh | 7 ++-- examples/test_32bit_gcc.sh | 7 ++-- examples/test_cmp_clang.sh | 10 ----- examples/test_cmp_gcc.sh | 10 ----- examples/test_extend.sh | 49 ------------------------ examples/test_initseed.sh | 8 ++-- examples/test_integerate.sh | 57 ++++++++++++++++++++++++++++ examples/test_linear_clang.sh | 9 +++++ examples/test_linear_gcc.sh | 9 +++++ examples/test_loop.sh | 5 ++- examples/test_monoton_clang.sh | 4 +- examples/test_monoton_gcc.sh | 4 +- examples/test_motiv.sh | 2 +- examples/test_nested.sh | 4 +- examples/test_no_fork_server.sh | 7 ++-- examples/test_stdin.sh | 8 ++++ examples/test_timeout.sh | 4 +- examples/timeout.c | 4 -- 24 files changed, 178 insertions(+), 115 deletions(-) rename examples/{extend.c => length.c} (89%) rename examples/{cmp.c => linear.c} (94%) create mode 100644 examples/stdin.c delete mode 100755 examples/test_cmp_clang.sh delete mode 100755 examples/test_cmp_gcc.sh delete mode 100755 examples/test_extend.sh create mode 100755 examples/test_integerate.sh create mode 100755 examples/test_linear_clang.sh create mode 100755 examples/test_linear_gcc.sh create mode 100755 examples/test_stdin.sh diff --git a/examples/extend.c b/examples/length.c similarity index 89% rename from examples/extend.c rename to examples/length.c index c5c9e76..6dd0688 100644 --- a/examples/extend.c +++ b/examples/length.c @@ -1,5 +1,4 @@ -/* A simple example to test whether Eclipser can cooperate well with AFL. */ - +// A simple example to test whether Eclipser can cooperate well with AFL. #include #include #include diff --git a/examples/cmp.c b/examples/linear.c similarity index 94% rename from examples/cmp.c rename to examples/linear.c index 957b011..c69848d 100644 --- a/examples/cmp.c +++ b/examples/linear.c @@ -1,7 +1,4 @@ -/* A simple example to test whether Eclipser can solve equality checking - * conditions. - */ - +// A simple example to test whether Eclipser can solve linear branch conditions. #include #include #include diff --git a/examples/loop.c b/examples/loop.c index d92a417..ef5e296 100644 --- a/examples/loop.c +++ b/examples/loop.c @@ -1,7 +1,3 @@ -/* A variatn of cmp.c example, to test if Eclipser can handle programs that - * have path explosion. Eclipser should not get stuck or raises exception, and - * pass '(i3 == 0x61626364)' condition when enough time is given. - */ #include #include #include diff --git a/examples/monoton.c b/examples/monoton.c index e9d2547..49b98c5 100644 --- a/examples/monoton.c +++ b/examples/monoton.c @@ -1,5 +1,3 @@ -/* Ab example with custom strcmp() function that requires binary search. */ - #include #include #include diff --git a/examples/nested.c b/examples/nested.c index 054dd8a..c8b63a7 100644 --- a/examples/nested.c +++ b/examples/nested.c @@ -1,7 +1,3 @@ -/* A simple example to test whether Eclipser can solve nested inequality - * checking conditions. - */ - #include #include #include diff --git a/examples/stdin.c b/examples/stdin.c new file mode 100644 index 0000000..2f911dc --- /dev/null +++ b/examples/stdin.c @@ -0,0 +1,67 @@ +#include +#include +#include +#include +#include + +/* Initialize '*_const' variables with a separate function, in order to prevent + * constant propagation optimization + */ +void init_consts(char *c, short* s, int* i, int64_t * i64) { + *c = 0x61; + *s = 0x6162; + *i = 0x61626364; + *i64 = 0x6162636465666768; +} + +int main(int argc, char ** argv){ + char c, c_const; + short s, s_const; + int i, i_const, i_be; + int64_t i64, i64_const; + + init_consts(&c_const, &s_const, &i_const, &i64_const); + + read(0, &c, sizeof(c)); + if (c == 0x41) { + printf("Found new path 1-1!\n"); + } + if (c == c_const) { // c_const is 0x61 + printf("Found new path 1-2!\n"); + } + + read(0, &s, sizeof(short)); + if (s == 0x4142) { + printf("Found new path 2-1!\n"); + } + if (s == s_const) { // s_const is 0x6162 + printf("Found new path 2-2!\n"); + } + + read(0, &i, sizeof(int)); + if (i == 0x41424344) { + printf("Found new path 3-1!\n"); + } + if (i == i_const) { // i_const is 0x61626364 + printf("Found new path 3-2!\n"); + } + + read(0, &i64, sizeof(int64_t)); + if (i64 == 0x4142434445464748ll) { + printf("Found new path 4-1!\n"); + } + if (i64 == i64_const) { // i64_const is 0x6162636465666768 + printf("Found new path 4-2!\n"); + } + + i_be = ((i >> 24) & 0xff) | // move byte 3 to byte 0 + ((i >> 8) & 0xff00) | // move byte 2 to byte 1 + ((i << 8) & 0xff0000) | // move byte 1 to byte 2 + ((i << 24)& 0xff000000); // byte 0 to byte 3 + + if (i_be == 0x71727374) { + printf("Found new path 5!\n"); + } + + return 0; +} diff --git a/examples/test_32bit_clang.sh b/examples/test_32bit_clang.sh index ad89884..6f6c44a 100755 --- a/examples/test_32bit_clang.sh +++ b/examples/test_32bit_clang.sh @@ -1,10 +1,9 @@ #!/bin/bash -# Grey-box concolic should find more than 7 file input test cases that have node -# coverage gain. -clang cmp.c -o cmp.bin -static -g -m32 || exit 1 +# Tests if Eclipser can solve linear branch conditions in a 32-bit binary. +clang linear.c -o linear.bin -static -g -m32 || exit 1 rm -rf box mkdir box cd box dotnet ../../build/Eclipser.dll \ - -p ../cmp.bin -t 5 -v 1 -o output -f input --arg input --architecture x86 + -p ../linear.bin -t 5 -v 1 -o output -f input --arg input --architecture x86 diff --git a/examples/test_32bit_gcc.sh b/examples/test_32bit_gcc.sh index 08679c7..deb524d 100755 --- a/examples/test_32bit_gcc.sh +++ b/examples/test_32bit_gcc.sh @@ -1,10 +1,9 @@ #!/bin/bash -# Grey-box concolic should find more than 7 file input test cases that have node -# coverage gain. -gcc cmp.c -o cmp.bin -static -g -m32 || exit 1 +# Tests if Eclipser can solve linear branch conditions in a 32-bit binary. +gcc linear.c -o linear.bin -static -g -m32 || exit 1 rm -rf box mkdir box cd box dotnet ../../build/Eclipser.dll \ - -p ../cmp.bin -t 5 -v 1 -o output -f input --arg input --architecture x86 + -p ../linear.bin -t 5 -v 1 -o output -f input --arg input --architecture x86 diff --git a/examples/test_cmp_clang.sh b/examples/test_cmp_clang.sh deleted file mode 100755 index 6fa17da..0000000 --- a/examples/test_cmp_clang.sh +++ /dev/null @@ -1,10 +0,0 @@ -#!/bin/bash - -# Grey-box concolic should find more than 9 file input test cases that have node -# coverage gain. -clang cmp.c -o cmp.bin -static -g || exit 1 -rm -rf box -mkdir box -cd box -dotnet ../../build/Eclipser.dll \ - -p ../cmp.bin -t 5 -v 1 -o output -f input --arg input diff --git a/examples/test_cmp_gcc.sh b/examples/test_cmp_gcc.sh deleted file mode 100755 index 951786c..0000000 --- a/examples/test_cmp_gcc.sh +++ /dev/null @@ -1,10 +0,0 @@ -#!/bin/bash - -# Grey-box concolic should find more than 9 file input test cases that have node -# coverage gain. -gcc cmp.c -o cmp.bin -static -g || exit 1 -rm -rf box -mkdir box -cd box -dotnet ../../build/Eclipser.dll \ - -p ../cmp.bin -t 5 -v 1 -o output -f input --arg input diff --git a/examples/test_extend.sh b/examples/test_extend.sh deleted file mode 100755 index ef7df31..0000000 --- a/examples/test_extend.sh +++ /dev/null @@ -1,49 +0,0 @@ -#!/bin/bash - -if [ -z "$1" ] -then - echo "Should provide AFL root directory as argument" - exit 1 -fi - -if [ ! -d $1 ] -then - echo "Cannot find AFL root directory path $1" - exit 1 -fi - -gcc extend.c -o extend.bin -static -g || exit 1 -rm -rf box -mkdir box -cd box -mkdir input -echo "" > input/empty -mkdir afl-box -mkdir eclipser-box -mkdir syncdir - -# Launch master and slave AFLs. -cd afl-box -cp ../../extend.bin ./ -echo "Start master AFL" -CMD="timeout 20 $1/afl-fuzz -i ../input -o ../syncdir/ -M afl-master -Q -- ./extend.bin @@ > log.txt" -echo "#!/bin/bash" > run_master.sh -echo $CMD >> run_master.sh -chmod 755 run_master.sh -./run_master.sh & -echo "Launched master AFL" - -echo "Start slave AFL" -CMD="timeout 20 $1/afl-fuzz -i ../input -o ../syncdir/ -S afl-slave -Q -- ./extend.bin @@ > log.txt" -echo "#!/bin/bash" > run_slave.sh -echo $CMD >> run_slave.sh -chmod 755 run_slave.sh -./run_slave.sh & -echo "Launched slave AFL" - -# Now launch Eclipser. -cd ../eclipser-box -cp ../../extend.bin ./ -dotnet ../../../build/Eclipser.dll \ - -t 20 -v 1 -s ../syncdir -o ../syncdir/eclipser-output \ - -p ./extend.bin --arg input -f input diff --git a/examples/test_initseed.sh b/examples/test_initseed.sh index d1c3adb..86c70d4 100755 --- a/examples/test_initseed.sh +++ b/examples/test_initseed.sh @@ -1,12 +1,12 @@ #!/bin/bash -# Grey-box concolic should find more than 9 file input test cases that have node -# coverage gain. -gcc cmp.c -o cmp.bin -static -g || exit 1 # -static option for easier debugging +# Tests if Eclipser can solve simple equality checking branch conditions when +# initial seeds are given. +gcc linear.c -o linear.bin -static -g || exit 1 rm -rf box mkdir box cd box mkdir seeds python -c 'print "B" * 16' > seeds/input dotnet ../../build/Eclipser.dll \ - -p ../cmp.bin -t 5 -v 1 -i seeds -o output -f input --arg input + -p ../linear.bin -t 5 -v 1 -i seeds -o output -f input --arg input diff --git a/examples/test_integerate.sh b/examples/test_integerate.sh new file mode 100755 index 0000000..482929f --- /dev/null +++ b/examples/test_integerate.sh @@ -0,0 +1,57 @@ +#!/bin/bash + +# Tests if Eclipser can cooperate well with AFL. First, Eclipser should be able +# to find a test case containing \x44\x43\x42\x41. Then, AFL should import this +# one and generate a new test case with an extended length. Finally, Eclipser +# should take this back, and find a crash that contains \x64\x63\x62\x61. + +if [ -z "$1" ] +then + echo "Should provide AFL root directory as an argument" + exit 1 +fi + +if [ ! -d $1 ] +then + echo "Cannot find AFL root directory path $1" + exit 1 +fi + +gcc length.c -o length.bin -static -g || exit 1 +rm -rf box +mkdir box +cd box +mkdir seeds +echo "" > seeds/empty +mkdir afl-master-box +mkdir afl-slave-box +mkdir eclipser-box +mkdir syncdir + +# Launch master and slave AFLs. +cd afl-master-box +cp ../../length.bin ./ +echo "Start master AFL" +CMD="timeout 20 $1/afl-fuzz -i ../seeds -o ../syncdir/ -M afl-master -f input -Q -- ./length.bin input > log.txt" +echo "#!/bin/bash" > run_master.sh +echo $CMD >> run_master.sh +chmod 755 run_master.sh +./run_master.sh & +echo "Launched master AFL" + +cd ../afl-slave-box +cp ../../length.bin ./ +echo "Start slave AFL" +CMD="timeout 20 $1/afl-fuzz -i ../seeds -o ../syncdir/ -S afl-slave -f input -Q -- ./length.bin input > log.txt" +echo "#!/bin/bash" > run_slave.sh +echo $CMD >> run_slave.sh +chmod 755 run_slave.sh +./run_slave.sh & +echo "Launched slave AFL" + +# Now launch Eclipser. +cd ../eclipser-box +cp ../../length.bin ./ +dotnet ../../../build/Eclipser.dll \ + -t 20 -v 1 -s ../syncdir -o ../syncdir/eclipser-output \ + -p ./length.bin --arg input -f input diff --git a/examples/test_linear_clang.sh b/examples/test_linear_clang.sh new file mode 100755 index 0000000..6096236 --- /dev/null +++ b/examples/test_linear_clang.sh @@ -0,0 +1,9 @@ +#!/bin/bash + +# Tests if Eclipser can solve linear branch conditions. +clang linear.c -o linear.bin -static -g || exit 1 +rm -rf box +mkdir box +cd box +dotnet ../../build/Eclipser.dll \ + -p ../linear.bin -t 5 -v 1 -o output -f input --arg input diff --git a/examples/test_linear_gcc.sh b/examples/test_linear_gcc.sh new file mode 100755 index 0000000..313e2cc --- /dev/null +++ b/examples/test_linear_gcc.sh @@ -0,0 +1,9 @@ +#!/bin/bash + +# Tests if Eclipser can solve linear branch conditions. +gcc linear.c -o linear.bin -static -g || exit 1 +rm -rf box +mkdir box +cd box +dotnet ../../build/Eclipser.dll \ + -p ../linear.bin -t 5 -v 1 -o output -f input --arg input diff --git a/examples/test_loop.sh b/examples/test_loop.sh index 79497ea..f32d494 100755 --- a/examples/test_loop.sh +++ b/examples/test_loop.sh @@ -1,9 +1,10 @@ #!/bin/bash -# Grey-box concolic should find file input test case with \x64\x63\x62\x61. +# Tests if Eclipser can handle path explosion in a loop. Eclipser should be able +# to find a test case containing \x64\x63\x62\x61. gcc loop.c -o loop.bin -static -g || exit 1 rm -rf box mkdir box cd box dotnet ../../build/Eclipser.dll \ - -p ../loop.bin -t 90 -v 1 -o output -f input --arg input --nsolve 10 + -p ../loop.bin -t 45 -v 1 -o output -f input --arg input --nsolve 10 diff --git a/examples/test_monoton_clang.sh b/examples/test_monoton_clang.sh index 5b90e30..3950f8a 100755 --- a/examples/test_monoton_clang.sh +++ b/examples/test_monoton_clang.sh @@ -1,7 +1,7 @@ #!/bin/bash -# Grey-box concolic should find test cases that have \x41\x42, \x61\x62, -# "Good!", and "Bad!". +# Tests if Eclipser can solve monotonic branch conditions. Eclipser should be +# able to find test cases containing \x41\x42, \x61\x62, "Good!", and "Bad!". clang monoton.c -o monoton.bin -static -g || exit 1 rm -rf box mkdir box diff --git a/examples/test_monoton_gcc.sh b/examples/test_monoton_gcc.sh index b91ceb3..4bf47ba 100755 --- a/examples/test_monoton_gcc.sh +++ b/examples/test_monoton_gcc.sh @@ -1,7 +1,7 @@ #!/bin/bash -# Grey-box concolic should find test cases that have \x41\x42, \x61\x62, -# "Good!", and "Bad!". +# Tests if Eclipser can solve monotonic branch conditions. Eclipser should be +# able to find test cases containing \x41\x42, \x61\x62, "Good!", and "Bad!". gcc monoton.c -o monoton.bin -static -g || exit 1 rm -rf box mkdir box diff --git a/examples/test_motiv.sh b/examples/test_motiv.sh index 32d831c..9fa2cca 100755 --- a/examples/test_motiv.sh +++ b/examples/test_motiv.sh @@ -1,6 +1,6 @@ #!/bin/bash -# Grey-box concolic should find a program-crashing test case within few seconds. +# Motivating example in the paper. Eclipser should find a crash immediately. gcc motiv.c -o motiv.bin -static -g || exit 1 rm -rf box mkdir box diff --git a/examples/test_nested.sh b/examples/test_nested.sh index eb9c6d3..6f9f5f5 100755 --- a/examples/test_nested.sh +++ b/examples/test_nested.sh @@ -1,7 +1,7 @@ #!/bin/bash -# Grey-box concolic should find test cases that have \x53, \x72\x71, and -# \x41\x42\x43\x44. +# Tests if Eclipser can handle nested conditional branches. Eclipser should be +# able to find test cases containing \x53, \x72\x71, and \x41\x42\x43\x44. gcc nested.c -o nested.bin -static -g || exit 1 rm -rf box mkdir box diff --git a/examples/test_no_fork_server.sh b/examples/test_no_fork_server.sh index e75f5e9..b185245 100755 --- a/examples/test_no_fork_server.sh +++ b/examples/test_no_fork_server.sh @@ -1,10 +1,9 @@ #!/bin/bash -# Grey-box concolic should find more than 9 file input test cases that have node -# coverage gain. -gcc cmp.c -o cmp.bin -static -g || exit 1 +# Tests if Eclipser can run without fork server as well. +gcc linear.c -o linear.bin -static -g || exit 1 rm -rf box mkdir box cd box dotnet ../../build/Eclipser.dll \ - -p ../cmp.bin -t 5 -v 1 -o output -f input --arg input --noforkserver + -p ../linear.bin -t 5 -v 1 -o output -f input --arg input --noforkserver diff --git a/examples/test_stdin.sh b/examples/test_stdin.sh new file mode 100755 index 0000000..b8b6a08 --- /dev/null +++ b/examples/test_stdin.sh @@ -0,0 +1,8 @@ +#!/bin/bash + +# Tests if Eclipser can fuzz standard input. +gcc stdin.c -o stdin.bin -static -g || exit 1 +rm -rf box +mkdir box +cd box +dotnet ../../build/Eclipser.dll -p ../stdin.bin -t 5 -v 1 -o output diff --git a/examples/test_timeout.sh b/examples/test_timeout.sh index f1f60ad..54db814 100755 --- a/examples/test_timeout.sh +++ b/examples/test_timeout.sh @@ -1,6 +1,8 @@ #!/bin/bash -# Grey-box concolic should find file input test case with \x44\x43\x42\x41. +# Tests if Eclipser can correctly identify coverage gain even if the program +# exits with a timeout. Eclipser should be able to find a test case containing +# \x44\x43\x42\x41. gcc timeout.c -o timeout.bin -static -g || exit 1 rm -rf box mkdir box diff --git a/examples/timeout.c b/examples/timeout.c index a103861..ce62e0f 100644 --- a/examples/timeout.c +++ b/examples/timeout.c @@ -1,7 +1,3 @@ -/* A variatn of cmp.c example, to test if Eclipser can handle programs that - * raises timeout. - */ - #include #include #include From dc8d265dda46e9fbbe370c78be13e9b2b90beac0 Mon Sep 17 00:00:00 2001 From: Jaeseung Choi Date: Fri, 16 Oct 2020 11:06:22 +0900 Subject: [PATCH 25/29] Make time limit option non-mandatory If timeout option (-t) is not given, run the fuzzer infinitely. --- src/Core/Options.fs | 8 ++++---- src/Fuzz/Fuzz.fs | 18 ++++++++++++------ 2 files changed, 16 insertions(+), 10 deletions(-) diff --git a/src/Core/Options.fs b/src/Core/Options.fs index b1134b8..8b91298 100644 --- a/src/Core/Options.fs +++ b/src/Core/Options.fs @@ -6,7 +6,7 @@ open Config type FuzzerCLI = | [] [] Verbose of int - | [] [] [] Timelimit of sec: int + | [] [] Timelimit of sec: int | [] [] [] OutputDir of path: string | [] [] SyncDir of path: string // Options related to program execution. @@ -32,11 +32,11 @@ with // Options related to program execution. | Program _ -> "Target program for test case generation with fuzzing." | ExecTimeout _ -> "Execution timeout (ms) for a fuzz run (default:500)" - | Architecture _ -> "Target program architecture (X86|X64) (default:X64)" + | Architecture _ -> "Target program architecture (x86|x64) (default:x64)" | NoForkServer -> "Do not use fork server for target program execution" // Options related to seed. | InputDir _ -> "Directory containing initial seeds." - | Arg _ -> "Command-line argument of program under test." + | Arg _ -> "Command-line argument of the target program to fuzz." | Filepath _ -> "File input's (fixed) path" // Options related to grey-box concolic testing technique. | NSolve _ -> "Number of branches to flip in grey-box concolic testing " + @@ -70,7 +70,7 @@ let parseFuzzOption (args: string array) = let r = try parser.Parse(args) with :? Argu.ArguParseException -> printLine (parser.PrintUsage()); exit 1 { Verbosity = r.GetResult (<@ Verbose @>, defaultValue = 0) - Timelimit = r.GetResult (<@ Timelimit @>) + Timelimit = r.GetResult (<@ Timelimit @>, defaultValue = -1) OutDir = r.GetResult (<@ OutputDir @>) SyncDir = r.GetResult (<@ SyncDir @>, defaultValue = "") // Options related to program execution. diff --git a/src/Fuzz/Fuzz.fs b/src/Fuzz/Fuzz.fs index 7825a88..1334ee6 100644 --- a/src/Fuzz/Fuzz.fs +++ b/src/Fuzz/Fuzz.fs @@ -67,10 +67,10 @@ let rec private fuzzLoop opt seedQueue n = let seedQueue = List.fold SeedQueue.enqueue seedQueue steppedItems fuzzLoop opt seedQueue (n + 1) -let private fuzzingTimer timeoutSec = async { - let timespan = System.TimeSpan(0, 0, 0, timeoutSec) - System.Threading.Thread.Sleep(timespan ) - printLine "Fuzzing timeout expired." +let private terminator timelimitSec = async { + let timespan = System.TimeSpan(0, 0, 0, timelimitSec) + System.Threading.Thread.Sleep(timespan) + log "[*] Fuzzing timeout expired." log "===== Statistics =====" TestCase.printStatistics () log "Done, clean up and exit..." @@ -78,13 +78,19 @@ let private fuzzingTimer timeoutSec = async { exit (0) } +let private setTimer opt = + if opt.Timelimit > 0 then + log "[*] Time limit : %d sec" opt.Timelimit + Async.Start (terminator opt.Timelimit) + else + log "[*] No time limit given, run infinitely" + [] let main args = let opt = parseFuzzOption args validateFuzzOption opt assertFileExists opt.TargetProg log "[*] Fuzz target : %s" opt.TargetProg - log "[*] Time limit : %d sec" opt.Timelimit createDirectoryIfNotExists opt.OutDir TestCase.initialize opt.OutDir Executor.initialize opt @@ -94,7 +100,7 @@ let main args = log "[*] Total %d initial seeds" (List.length initialSeeds) let initItems = List.choose (makeInitialItems opt) initialSeeds let initQueue = List.fold SeedQueue.enqueue emptyQueue initItems - Async.Start (fuzzingTimer opt.Timelimit) + setTimer opt log "[*] Start fuzzing" fuzzLoop opt initQueue 0 0 // Unreachable From d77763b15e93a65c67960431bba3bf89f53fc0b0 Mon Sep 17 00:00:00 2001 From: Jaeseung Choi Date: Fri, 16 Oct 2020 00:00:31 -0700 Subject: [PATCH 26/29] Polish synchronization and scheduling - Slightly defer the first synchronization with AFL. - When calculating testing efficiency, exclude execution counts for seed queue initialization and synchronization. --- src/Core/Executor.fs | 21 +++++++++++++++------ src/Fuzz/Fuzz.fs | 4 ++-- src/Fuzz/Scheduler.fs | 10 +++++----- src/Fuzz/Sync.fs | 7 ++++++- src/Fuzz/TestCase.fs | 11 ++++++++--- 5 files changed, 36 insertions(+), 17 deletions(-) diff --git a/src/Core/Executor.fs b/src/Core/Executor.fs index 185ed8a..26559c9 100644 --- a/src/Core/Executor.fs +++ b/src/Core/Executor.fs @@ -24,7 +24,8 @@ let mutable private coverageLog = "" let mutable private bitmapLog = "" let mutable private dbgLog = "" let mutable private forkServerOn = false -let mutable private roundExecutions = 0 +let mutable private roundStatisticsOn = false +let mutable private roundExecs = 0 (*** Tracer and file paths ***) @@ -110,9 +111,17 @@ let cleanup () = (*** Execution count statistics ***) -let getRoundExecutions () = roundExecutions +let enableRoundStatistics () = roundStatisticsOn <- true -let resetRoundExecutions () = roundExecutions <- 0 +let disableRoundStatistics () = roundStatisticsOn <- false + +let getRoundExecs () = roundExecs + +// Increment only if roundStatisticsOn flag is set. We don't want the executions +// for the synchronization with AFL to affect the efficiency calculation. +let incrRoundExecs () = if roundStatisticsOn then roundExecs <- roundExecs + 1 + +let resetRoundExecs () = roundExecs <- 0 (*** Setup functions ***) @@ -193,7 +202,7 @@ let private tryReadBranchInfo opt filename tryVal = (*** Tracer execution functions ***) let private runTracer tracerType opt (stdin: byte array) = - roundExecutions <- roundExecutions + 1 + incrRoundExecs () let targetProg = opt.TargetProg let timeout = opt.ExecTimeout let tracer = selectTracer tracerType opt.Architecture @@ -203,7 +212,7 @@ let private runTracer tracerType opt (stdin: byte array) = exec(argc, args, stdin.Length, stdin, timeout) let private runCoverageTracerForked opt stdin = - roundExecutions <- roundExecutions + 1 + incrRoundExecs () let timeout = opt.ExecTimeout let stdLen = Array.length stdin let signal = exec_fork_coverage(timeout, stdLen, stdin) @@ -211,7 +220,7 @@ let private runCoverageTracerForked opt stdin = signal let private runBranchTracerForked opt stdin addr idx covMeasure = - roundExecutions <- roundExecutions + 1 + incrRoundExecs () let timeout = opt.ExecTimeout let stdLen = Array.length stdin let covEnum = CoverageMeasure.toEnum covMeasure diff --git a/src/Fuzz/Fuzz.fs b/src/Fuzz/Fuzz.fs index 1334ee6..c00c1c4 100644 --- a/src/Fuzz/Fuzz.fs +++ b/src/Fuzz/Fuzz.fs @@ -94,13 +94,13 @@ let main args = createDirectoryIfNotExists opt.OutDir TestCase.initialize opt.OutDir Executor.initialize opt - Scheduler.initialize () let emptyQueue = SeedQueue.initialize () let initialSeeds = initializeSeeds opt log "[*] Total %d initial seeds" (List.length initialSeeds) let initItems = List.choose (makeInitialItems opt) initialSeeds let initQueue = List.fold SeedQueue.enqueue emptyQueue initItems setTimer opt + Scheduler.initialize () // Should be called after preprocessing initial seeds. log "[*] Start fuzzing" - fuzzLoop opt initQueue 0 + fuzzLoop opt initQueue 1 // Start from 1, to slightly defer the first sync. 0 // Unreachable diff --git a/src/Fuzz/Scheduler.fs b/src/Fuzz/Scheduler.fs index b7f0a13..996a8bd 100644 --- a/src/Fuzz/Scheduler.fs +++ b/src/Fuzz/Scheduler.fs @@ -21,17 +21,17 @@ let private decideSleepFactor roundExecs roundTCs = max SLEEP_FACTOR_MIN (min SLEEP_FACTOR_MAX factor) let initialize () = - Executor.resetRoundExecutions() - TestCase.resetRoundTestCaseCount() + Executor.enableRoundStatistics() + TestCase.enableRoundStatistics() timer.Start() // Check the efficiency of the system and sleep for a while to adjust the weight // of resource use with AFL. -let checkAndReserveTime () = - let roundExecs = Executor.getRoundExecutions () +let checkAndReserveTime opt = + let roundExecs = Executor.getRoundExecs () if roundExecs > ROUND_SIZE then let roundTCs = TestCase.getRoundTestCaseCount () - Executor.resetRoundExecutions() + Executor.resetRoundExecs() TestCase.resetRoundTestCaseCount() let sleepFactor = decideSleepFactor roundExecs roundTCs let roundElapsed = timer.ElapsedMilliseconds diff --git a/src/Fuzz/Sync.fs b/src/Fuzz/Sync.fs index e712894..ee9269b 100644 --- a/src/Fuzz/Sync.fs +++ b/src/Fuzz/Sync.fs @@ -46,4 +46,9 @@ let run opt seedQueue = let subDirs = Directory.EnumerateDirectories(syncDir) |> List.ofSeq |> List.map (fun dirName -> Path.Combine(syncDir, dirName)) |> List.filter (fun d -> d <> outDir) // Exclude our own output. - List.fold (syncFromDir opt) seedQueue subDirs + Executor.disableRoundStatistics() + TestCase.disableRoundStatistics() + let newSeedQueue = List.fold (syncFromDir opt) seedQueue subDirs + Executor.enableRoundStatistics() + TestCase.enableRoundStatistics() + newSeedQueue diff --git a/src/Fuzz/TestCase.fs b/src/Fuzz/TestCase.fs index 88c710c..ab8495c 100644 --- a/src/Fuzz/TestCase.fs +++ b/src/Fuzz/TestCase.fs @@ -22,6 +22,7 @@ let mutable private totalFPEs = 0 let mutable private totalAborts = 0 let mutable private totalCrashes = 0 let mutable private totalTestCases = 0 +let mutable private roundStatisticsOn = false let mutable private roundTestCases = 0 let printStatistics () = @@ -41,12 +42,16 @@ let private incrCrashCount exitSig = | _ -> failwith "updateCrashCount() called with a non-crashing exit signal" totalCrashes <- totalCrashes + 1 -let private incrTestCaseCount () = - totalTestCases <- totalTestCases + 1 - roundTestCases <- roundTestCases + 1 +let enableRoundStatistics () = roundStatisticsOn <- true + +let disableRoundStatistics () = roundStatisticsOn <- false let getRoundTestCaseCount () = roundTestCases +let incrTestCaseCount () = + totalTestCases <- totalTestCases + 1 + if roundStatisticsOn then roundTestCases <- roundTestCases + 1 + let resetRoundTestCaseCount () = roundTestCases <- 0 (*** Test case storing functions ***) From 932104b154d17235c8e0690b63621c5fb71d396e Mon Sep 17 00:00:00 2001 From: Jaeseung Choi Date: Fri, 16 Oct 2020 00:00:51 -0700 Subject: [PATCH 27/29] Polish log messages and verbosity behavior Fix log message formats to be consistent. Make some frequent messages to appear only when verbosity >= 2. --- examples/test_32bit_clang.sh | 2 +- examples/test_32bit_gcc.sh | 2 +- examples/test_initseed.sh | 2 +- examples/test_integerate.sh | 2 +- examples/test_linear_clang.sh | 2 +- examples/test_linear_gcc.sh | 2 +- examples/test_loop.sh | 2 +- examples/test_monoton_clang.sh | 2 +- examples/test_monoton_gcc.sh | 2 +- examples/test_motiv.sh | 2 +- examples/test_nested.sh | 2 +- examples/test_no_fork_server.sh | 2 +- examples/test_stdin.sh | 2 +- examples/test_timeout.sh | 2 +- src/Core/Executor.fs | 3 --- src/Core/Options.fs | 2 +- src/Fuzz/Fuzz.fs | 15 +++++++-------- src/Fuzz/Scheduler.fs | 17 +++++++++-------- src/Fuzz/Sync.fs | 2 +- src/Fuzz/TestCase.fs | 2 +- 20 files changed, 33 insertions(+), 36 deletions(-) diff --git a/examples/test_32bit_clang.sh b/examples/test_32bit_clang.sh index 6f6c44a..519754f 100755 --- a/examples/test_32bit_clang.sh +++ b/examples/test_32bit_clang.sh @@ -6,4 +6,4 @@ rm -rf box mkdir box cd box dotnet ../../build/Eclipser.dll \ - -p ../linear.bin -t 5 -v 1 -o output -f input --arg input --architecture x86 + -p ../linear.bin -t 5 -v 2 -o output -f input --arg input --architecture x86 diff --git a/examples/test_32bit_gcc.sh b/examples/test_32bit_gcc.sh index deb524d..fc6a285 100755 --- a/examples/test_32bit_gcc.sh +++ b/examples/test_32bit_gcc.sh @@ -6,4 +6,4 @@ rm -rf box mkdir box cd box dotnet ../../build/Eclipser.dll \ - -p ../linear.bin -t 5 -v 1 -o output -f input --arg input --architecture x86 + -p ../linear.bin -t 5 -v 2 -o output -f input --arg input --architecture x86 diff --git a/examples/test_initseed.sh b/examples/test_initseed.sh index 86c70d4..32c9bb7 100755 --- a/examples/test_initseed.sh +++ b/examples/test_initseed.sh @@ -9,4 +9,4 @@ cd box mkdir seeds python -c 'print "B" * 16' > seeds/input dotnet ../../build/Eclipser.dll \ - -p ../linear.bin -t 5 -v 1 -i seeds -o output -f input --arg input + -p ../linear.bin -t 5 -v 2 -i seeds -o output -f input --arg input diff --git a/examples/test_integerate.sh b/examples/test_integerate.sh index 482929f..ed0b2f2 100755 --- a/examples/test_integerate.sh +++ b/examples/test_integerate.sh @@ -53,5 +53,5 @@ echo "Launched slave AFL" cd ../eclipser-box cp ../../length.bin ./ dotnet ../../../build/Eclipser.dll \ - -t 20 -v 1 -s ../syncdir -o ../syncdir/eclipser-output \ + -t 20 -v 2 -s ../syncdir -o ../syncdir/eclipser-output \ -p ./length.bin --arg input -f input diff --git a/examples/test_linear_clang.sh b/examples/test_linear_clang.sh index 6096236..7d406da 100755 --- a/examples/test_linear_clang.sh +++ b/examples/test_linear_clang.sh @@ -6,4 +6,4 @@ rm -rf box mkdir box cd box dotnet ../../build/Eclipser.dll \ - -p ../linear.bin -t 5 -v 1 -o output -f input --arg input + -p ../linear.bin -t 5 -v 2 -o output -f input --arg input diff --git a/examples/test_linear_gcc.sh b/examples/test_linear_gcc.sh index 313e2cc..3295e36 100755 --- a/examples/test_linear_gcc.sh +++ b/examples/test_linear_gcc.sh @@ -6,4 +6,4 @@ rm -rf box mkdir box cd box dotnet ../../build/Eclipser.dll \ - -p ../linear.bin -t 5 -v 1 -o output -f input --arg input + -p ../linear.bin -t 5 -v 2 -o output -f input --arg input diff --git a/examples/test_loop.sh b/examples/test_loop.sh index f32d494..c082679 100755 --- a/examples/test_loop.sh +++ b/examples/test_loop.sh @@ -7,4 +7,4 @@ rm -rf box mkdir box cd box dotnet ../../build/Eclipser.dll \ - -p ../loop.bin -t 45 -v 1 -o output -f input --arg input --nsolve 10 + -p ../loop.bin -t 45 -v 2 -o output -f input --arg input --nsolve 10 diff --git a/examples/test_monoton_clang.sh b/examples/test_monoton_clang.sh index 3950f8a..d019b9e 100755 --- a/examples/test_monoton_clang.sh +++ b/examples/test_monoton_clang.sh @@ -7,4 +7,4 @@ rm -rf box mkdir box cd box dotnet ../../build/Eclipser.dll \ - -p ../monoton.bin -t 10 -v 1 -o output -f input --arg input + -p ../monoton.bin -t 10 -v 2 -o output -f input --arg input diff --git a/examples/test_monoton_gcc.sh b/examples/test_monoton_gcc.sh index 4bf47ba..9846532 100755 --- a/examples/test_monoton_gcc.sh +++ b/examples/test_monoton_gcc.sh @@ -7,4 +7,4 @@ rm -rf box mkdir box cd box dotnet ../../build/Eclipser.dll \ - -p ../monoton.bin -t 10 -v 1 -o output -f input --arg input + -p ../monoton.bin -t 10 -v 2 -o output -f input --arg input diff --git a/examples/test_motiv.sh b/examples/test_motiv.sh index 9fa2cca..f241eba 100755 --- a/examples/test_motiv.sh +++ b/examples/test_motiv.sh @@ -6,4 +6,4 @@ rm -rf box mkdir box cd box dotnet ../../build/Eclipser.dll \ - -p ../motiv.bin -t 5 -v 1 -o output -f input --arg input + -p ../motiv.bin -t 5 -v 2 -o output -f input --arg input diff --git a/examples/test_nested.sh b/examples/test_nested.sh index 6f9f5f5..0448d37 100755 --- a/examples/test_nested.sh +++ b/examples/test_nested.sh @@ -7,4 +7,4 @@ rm -rf box mkdir box cd box dotnet ../../build/Eclipser.dll \ - -p ../nested.bin -t 20 -v 1 -o output -f input --arg input + -p ../nested.bin -t 20 -v 2 -o output -f input --arg input diff --git a/examples/test_no_fork_server.sh b/examples/test_no_fork_server.sh index b185245..7abe9ec 100755 --- a/examples/test_no_fork_server.sh +++ b/examples/test_no_fork_server.sh @@ -6,4 +6,4 @@ rm -rf box mkdir box cd box dotnet ../../build/Eclipser.dll \ - -p ../linear.bin -t 5 -v 1 -o output -f input --arg input --noforkserver + -p ../linear.bin -t 5 -v 2 -o output -f input --arg input --noforkserver diff --git a/examples/test_stdin.sh b/examples/test_stdin.sh index b8b6a08..18c7b76 100755 --- a/examples/test_stdin.sh +++ b/examples/test_stdin.sh @@ -5,4 +5,4 @@ gcc stdin.c -o stdin.bin -static -g || exit 1 rm -rf box mkdir box cd box -dotnet ../../build/Eclipser.dll -p ../stdin.bin -t 5 -v 1 -o output +dotnet ../../build/Eclipser.dll -p ../stdin.bin -t 5 -v 2 -o output diff --git a/examples/test_timeout.sh b/examples/test_timeout.sh index 54db814..ae034c8 100755 --- a/examples/test_timeout.sh +++ b/examples/test_timeout.sh @@ -8,4 +8,4 @@ rm -rf box mkdir box cd box dotnet ../../build/Eclipser.dll \ - -p ../timeout.bin -t 5 -v 1 -o output -f input --arg input + -p ../timeout.bin -t 5 -v 2 -o output -f input --arg input diff --git a/src/Core/Executor.fs b/src/Core/Executor.fs index 26559c9..4693324 100644 --- a/src/Core/Executor.fs +++ b/src/Core/Executor.fs @@ -76,7 +76,6 @@ let private initializeForkServer opt = let initialize opt = let outDir = opt.OutDir - let verbosity = opt.Verbosity // Set environment variables for the instrumentor. branchLog <- System.IO.Path.Combine(outDir, ".branch") coverageLog <- System.IO.Path.Combine(outDir, ".coverage") @@ -87,8 +86,6 @@ let initialize opt = use bitmapFile = File.Create(bitmapLog) bitmapFile.SetLength(BITMAP_SIZE) set_env("ECL_BITMAP_LOG", System.IO.Path.GetFullPath(bitmapLog)) - if verbosity >= 2 then - set_env("ECL_DBG_LOG", System.IO.Path.GetFullPath(dbgLog)) initialize_exec () if opt.ForkServer then set_env("ECL_FORK_SERVER", "1") diff --git a/src/Core/Options.fs b/src/Core/Options.fs index 8b91298..dd694b5 100644 --- a/src/Core/Options.fs +++ b/src/Core/Options.fs @@ -69,7 +69,7 @@ let parseFuzzOption (args: string array) = let parser = ArgumentParser.Create (programName = cmdPrefix) let r = try parser.Parse(args) with :? Argu.ArguParseException -> printLine (parser.PrintUsage()); exit 1 - { Verbosity = r.GetResult (<@ Verbose @>, defaultValue = 0) + { Verbosity = r.GetResult (<@ Verbose @>, defaultValue = 1) Timelimit = r.GetResult (<@ Timelimit @>, defaultValue = -1) OutDir = r.GetResult (<@ OutputDir @>) SyncDir = r.GetResult (<@ SyncDir @>, defaultValue = "") diff --git a/src/Fuzz/Fuzz.fs b/src/Fuzz/Fuzz.fs index c00c1c4..5aa4477 100644 --- a/src/Fuzz/Fuzz.fs +++ b/src/Fuzz/Fuzz.fs @@ -1,16 +1,15 @@ module Eclipser.Fuzz -open System.Threading open Config open Utils open Options -let private printFoundSeed verbosity seed = - if verbosity >= 1 then log "[*] Found a new seed: %s" (Seed.toString seed) +let private printFoundSeed opt seed = + if opt.Verbosity >= 1 then log "[*] Found a new seed: %s" (Seed.toString seed) let private evalSeed opt seed exitSig covGain = TestCase.save opt seed exitSig covGain - if covGain = NewEdge then printFoundSeed opt.Verbosity seed + if covGain = NewEdge then printFoundSeed opt seed let isAbnormal = Signal.isTimeout exitSig || Signal.isCrash exitSig if isAbnormal then None else Priority.ofCoverageGain covGain @@ -40,7 +39,7 @@ let private makeSteppedItems pr seed = // Decides how to share the resource with AFL instances. let private scheduleWithAFL opt = - if opt.SyncDir <> "" then Scheduler.checkAndReserveTime () + if opt.SyncDir <> "" then Scheduler.checkAndReserveTime opt // Sychronize the seed queue with AFL instances. let private syncWithAFL opt seedQueue n = @@ -51,12 +50,12 @@ let rec private fuzzLoop opt seedQueue n = scheduleWithAFL opt let seedQueue = syncWithAFL opt seedQueue n if SeedQueue.isEmpty seedQueue then - if n % 10 = 0 && opt.Verbosity >= 1 then log "Seed queue empty, waiting..." - Thread.Sleep(1000) + if n % 10 = 0 && opt.Verbosity >= 2 then log "Seed queue empty, waiting..." + System.Threading.Thread.Sleep(1000) fuzzLoop opt seedQueue (n + 1) else let priority, seed, seedQueue = SeedQueue.dequeue seedQueue - if opt.Verbosity >= 1 then log "Fuzzing with: %s" (Seed.toString seed) + if opt.Verbosity >= 2 then log "Fuzzing with: %s" (Seed.toString seed) let newItems = GreyConcolic.run seed opt // Relocate the cursors of newly generated seeds. let relocatedItems = makeRelocatedItems opt newItems diff --git a/src/Fuzz/Scheduler.fs b/src/Fuzz/Scheduler.fs index 996a8bd..eee394f 100644 --- a/src/Fuzz/Scheduler.fs +++ b/src/Fuzz/Scheduler.fs @@ -2,7 +2,7 @@ module Eclipser.Scheduler open Config open Utils -open System.Threading +open Options let private timer = new System.Diagnostics.Stopwatch() @@ -11,12 +11,12 @@ let private RAND_FUZZ_EFFICIENCY = 0.0005 // Decides sleep factor 'f', which will be used to sleep for 'f * elapsed time'. // This means we will utilize 1 / (2 * (f + 1)) of the system resource. -let private decideSleepFactor roundExecs roundTCs = +let private decideSleepFactor opt roundExecs roundTCs = let greyConcEfficiency = float (roundTCs) / float (roundExecs) + if opt.Verbosity >= 1 then log "[*] Efficiency = %.4f" greyConcEfficiency // GREY_CONC_EFF : RAND_FUZZ_EFF = 1 : 2 * factor + 1 let factor = if greyConcEfficiency = 0.0 then SLEEP_FACTOR_MAX else (RAND_FUZZ_EFFICIENCY / greyConcEfficiency - 1.0) / 2.0 - log "[*] Grey-concolic eff. = %.3f, factor = %.3f" greyConcEfficiency factor // Bound the factor between minimum and maximum value allowed. max SLEEP_FACTOR_MIN (min SLEEP_FACTOR_MAX factor) @@ -33,11 +33,12 @@ let checkAndReserveTime opt = let roundTCs = TestCase.getRoundTestCaseCount () Executor.resetRoundExecs() TestCase.resetRoundTestCaseCount() - let sleepFactor = decideSleepFactor roundExecs roundTCs + let sleepFactor = decideSleepFactor opt roundExecs roundTCs let roundElapsed = timer.ElapsedMilliseconds - log "[*] Elapsed round time: %d sec." (roundElapsed / 1000L) - timer.Reset() let sleepTime = int (float (roundElapsed) * sleepFactor) - log "[*] Decided sleep time: %d sec." (sleepTime / 1000) - Thread.Sleep(sleepTime) + if opt.Verbosity >= 1 then + log "[*] Elapsed round time: %d sec." (roundElapsed / 1000L) + log "[*] Decided sleep time: %d sec." (sleepTime / 1000) + System.Threading.Thread.Sleep(sleepTime) + timer.Reset() timer.Start() diff --git a/src/Fuzz/Sync.fs b/src/Fuzz/Sync.fs index ee9269b..12733ee 100644 --- a/src/Fuzz/Sync.fs +++ b/src/Fuzz/Sync.fs @@ -26,7 +26,7 @@ let private syncTestCase opt maxImport (accSeedQueue, accMaxImport) tcPath = | None -> (accSeedQueue, accMaxImport) | Some num when num <= maxImport -> (accSeedQueue, accMaxImport) | Some num -> // Unhandled test case ID. - log "Synchronizing seed queue with %s" tcPath + if opt.Verbosity >= 2 then log "Synchronizing seed queue with %s" tcPath let accMaxImport = if num > accMaxImport then num else accMaxImport let accSeedQueue = importSeed opt tcPath accSeedQueue (accSeedQueue, accMaxImport) diff --git a/src/Fuzz/TestCase.fs b/src/Fuzz/TestCase.fs index ab8495c..807e1e6 100644 --- a/src/Fuzz/TestCase.fs +++ b/src/Fuzz/TestCase.fs @@ -57,7 +57,7 @@ let resetRoundTestCaseCount () = roundTestCases <- 0 (*** Test case storing functions ***) let private dumpCrash opt seed exitSig = - if opt.Verbosity >= 0 then log "Save crash seed : %s" (Seed.toString seed) + if opt.Verbosity >= 1 then log "[*] Save crash seed : %s" (Seed.toString seed) let crashName = sprintf "id:%06d" totalCrashes let crashPath = System.IO.Path.Combine(crashDir, crashName) System.IO.File.WriteAllBytes(crashPath, Seed.concretize seed) From fc239a8c77208c90155a38e5167c9acc5057327b Mon Sep 17 00:00:00 2001 From: Jaeseung Choi Date: Fri, 16 Oct 2020 13:14:41 +0900 Subject: [PATCH 28/29] Introduce execution timeout decision logic To support a fair comparison with AFL, adopt AFL's idea of deciding execution timeout with initial seed set. Note that when execution timeout is explicitly given with -e option, Eclipser will just use it. --- src/Core/Config.fs | 8 ++++-- src/Core/Executor.fs | 3 +-- src/Core/Options.fs | 4 +-- src/Fuzz/Fuzz.fs | 63 +++++++++++++++++++++++++++++++++++-------- src/Fuzz/SeedQueue.fs | 2 +- 5 files changed, 62 insertions(+), 18 deletions(-) diff --git a/src/Core/Config.fs b/src/Core/Config.fs index 62c074c..f575cf3 100644 --- a/src/Core/Config.fs +++ b/src/Core/Config.fs @@ -13,8 +13,12 @@ let ROUND_SIZE = 10000 let SLEEP_FACTOR_MIN = 0.0 let SLEEP_FACTOR_MAX = 4.0 -/// Default execution timeout of target program. -let DEF_EXEC_TO = 500UL +/// Minimum and maximum value for the execution timeout of target program. Note +/// that AFL uses 1000UL for EXEC_TIMEOUT_MAX, but we use a higher value since +/// Eclipser is a binary-based fuzzer. Note that this range is ignored when an +/// explicit execution timeout is given with '-e' option. +let EXEC_TIMEOUT_MIN = 400UL +let EXEC_TIMEOUT_MAX = 4000UL /// Maximum length of chunk to try in grey-box concolic testing. let MAX_CHUNK_LEN = 10 diff --git a/src/Core/Executor.fs b/src/Core/Executor.fs index 4693324..cc39b93 100644 --- a/src/Core/Executor.fs +++ b/src/Core/Executor.fs @@ -265,8 +265,7 @@ let getBranchInfoOnly opt seed tryVal targPoint = if forkServerOn then runBranchTracerForked opt stdin addr idx Ignore else setEnvForBranch addr idx Ignore; runTracer Branch opt stdin |> ignore - let brInfoOpt = tryReadBranchInfo opt branchLog tryVal - brInfoOpt + tryReadBranchInfo opt branchLog tryVal let nativeExecute opt seed = let targetProg = opt.TargetProg diff --git a/src/Core/Options.fs b/src/Core/Options.fs index dd694b5..0ea5a70 100644 --- a/src/Core/Options.fs +++ b/src/Core/Options.fs @@ -11,7 +11,7 @@ type FuzzerCLI = | [] [] SyncDir of path: string // Options related to program execution. | [] [] [] Program of path: string - | [] ExecTimeout of millisec:uint64 + | [] [] ExecTimeout of millisec:uint64 | [] Architecture of string | [] NoForkServer // Options related to seed. @@ -75,7 +75,7 @@ let parseFuzzOption (args: string array) = SyncDir = r.GetResult (<@ SyncDir @>, defaultValue = "") // Options related to program execution. TargetProg = System.IO.Path.GetFullPath(r.GetResult (<@ Program @>)) - ExecTimeout = r.GetResult (<@ ExecTimeout @>, defaultValue = DEF_EXEC_TO) + ExecTimeout = r.GetResult (<@ ExecTimeout @>, defaultValue = 0UL) Architecture = r.GetResult(<@ Architecture @>, defaultValue = "X64") |> Arch.ofString ForkServer = not (r.Contains(<@ NoForkServer @>)) // Enable by default. diff --git a/src/Fuzz/Fuzz.fs b/src/Fuzz/Fuzz.fs index 5aa4477..95563d7 100644 --- a/src/Fuzz/Fuzz.fs +++ b/src/Fuzz/Fuzz.fs @@ -7,12 +7,6 @@ open Options let private printFoundSeed opt seed = if opt.Verbosity >= 1 then log "[*] Found a new seed: %s" (Seed.toString seed) -let private evalSeed opt seed exitSig covGain = - TestCase.save opt seed exitSig covGain - if covGain = NewEdge then printFoundSeed opt seed - let isAbnormal = Signal.isTimeout exitSig || Signal.isCrash exitSig - if isAbnormal then None else Priority.ofCoverageGain covGain - let private initializeSeeds opt = if opt.InputDir = "" then [Seed.make opt.FuzzSource] else System.IO.Directory.EnumerateFiles opt.InputDir // Obtain file list @@ -20,10 +14,58 @@ let private initializeSeeds opt = |> List.map System.IO.File.ReadAllBytes // Read in file contents |> List.map (Seed.makeWith opt.FuzzSource) // Create seed with content -let private makeInitialItems opt seed = +let private checkInitSeedCoverage opt seed = let exitSig, covGain = Executor.getCoverage opt seed TestCase.save opt seed exitSig covGain - Option.map (fun pr -> (pr, seed)) (Priority.ofCoverageGain covGain) + match Priority.ofCoverageGain covGain with + | None -> None + | Some priority -> Some (priority, seed) + +let private initializeQueue opt seeds = + // If the execution timeout is not given, use a large enough value for now. + let opt = if opt.ExecTimeout <> 0UL then opt + else { opt with ExecTimeout = EXEC_TIMEOUT_MAX } + let initItems = List.choose (checkInitSeedCoverage opt) seeds + List.fold SeedQueue.enqueue SeedQueue.empty initItems + +// Measure the execution time of an initial seed. Choose the longer time between +// getCoverage() and getBranchTrace() call. +let private getInitSeedExecTime opt seed = + let stopWatch = new System.Diagnostics.Stopwatch() + stopWatch.Start() + Executor.getCoverage opt seed |> ignore + let time1 = stopWatch.Elapsed.TotalMilliseconds + stopWatch.Reset() + stopWatch.Start() + Executor.getBranchTrace opt seed 0I |> ignore // Use a dummy 'tryVal' arg. + let time2 = stopWatch.Elapsed.TotalMilliseconds + max time1 time2 + +// Decide execution timeout based on the execution time of initial seeds. Adopt +// the basic idea from AFL, and adjust coefficients and range for Eclipser. +let private decideExecTimeout (execTimes: float list) = + let avgExecTime = List.average execTimes + let maxExecTime = List.max execTimes + log "[*] Initial seed execution time: avg = %.1f (ms), max = %.1f (ms)" + avgExecTime maxExecTime + let execTimeout = uint64 (max (4.0 * avgExecTime) (1.2 * maxExecTime)) + let execTimeout = min EXEC_TIMEOUT_MAX (max EXEC_TIMEOUT_MIN execTimeout) + log "[*] Set execution timeout to %d (ms)" execTimeout + execTimeout + +// If the execution timeout is not given, set it to a large enough value and +// find each seed's execution time. Then, decide a new timeout based on them. +let private updateExecTimeout opt seeds = + if opt.ExecTimeout <> 0UL then opt + else let opt = { opt with ExecTimeout = EXEC_TIMEOUT_MAX } + let execTimes = List.map (getInitSeedExecTime opt) seeds + { opt with ExecTimeout = decideExecTimeout execTimes } + +let private evalSeed opt seed exitSig covGain = + TestCase.save opt seed exitSig covGain + if covGain = NewEdge then printFoundSeed opt seed + let isAbnormal = Signal.isTimeout exitSig || Signal.isCrash exitSig + if isAbnormal then None else Priority.ofCoverageGain covGain let private makeRelocatedItems opt seeds = let collector (seed, exitSig, covGain) = @@ -93,11 +135,10 @@ let main args = createDirectoryIfNotExists opt.OutDir TestCase.initialize opt.OutDir Executor.initialize opt - let emptyQueue = SeedQueue.initialize () let initialSeeds = initializeSeeds opt log "[*] Total %d initial seeds" (List.length initialSeeds) - let initItems = List.choose (makeInitialItems opt) initialSeeds - let initQueue = List.fold SeedQueue.enqueue emptyQueue initItems + let initQueue = initializeQueue opt initialSeeds + let opt = updateExecTimeout opt initialSeeds setTimer opt Scheduler.initialize () // Should be called after preprocessing initial seeds. log "[*] Start fuzzing" diff --git a/src/Fuzz/SeedQueue.fs b/src/Fuzz/SeedQueue.fs index 029ae64..8d35b2c 100644 --- a/src/Fuzz/SeedQueue.fs +++ b/src/Fuzz/SeedQueue.fs @@ -42,7 +42,7 @@ type SeedQueue = { module SeedQueue = - let initialize () = + let empty = { Favoreds = Queue.empty Normals = Queue.empty } From ca070cee068c819709e427e86ebe1a064435a7ca Mon Sep 17 00:00:00 2001 From: Jaeseung Choi Date: Mon, 19 Oct 2020 00:22:15 -0700 Subject: [PATCH 29/29] Update README and CHANGELOG --- CHANGELOG.md | 17 +++++ README.md | 174 +++++++++++++++------------------------------------ 2 files changed, 69 insertions(+), 122 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 36a88b4..4322d54 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,22 @@ # Eclipser Change Log +## v2.0 + +* Simplify architecture by removing multiple input source fuzzing. This feature + has been supported for the comparison against KLEE. +* Remove our own random fuzzing module, and support integration with AFL. +* Fix QEMU instrumentation code (update to QEMU-2.10.0, fix bugs, optimize). +* Add a feature to decide execution timeout automatically. +* Clean up codes. +* Update command line interface. +* Update test examples. + +## v1.1 + +* Fix initial seed set handling. +* Use edge coverage instead of node coverage. +* Fix the default parameters for maximum file/stdin length. + ## v1.0 * Stop polluting '/tmp/' directory and keep the intermediate files internally. diff --git a/README.md b/README.md index 7780926..35d4a62 100644 --- a/README.md +++ b/README.md @@ -38,150 +38,80 @@ $ make # Usage -- Basic options +- Running with AFL -The basic usage of Eclipser is as follow. Note that you should provide `fuzz` -keyword before other options. +Starting from v2.0, Eclipser only performs grey-box concolic testing for test +case generation and relies on AFL to perform random-based fuzzing (for the +context of this decision, refer to [Eclipser v2.0](#eclipser-v20) section +below). Therefore, you should first launch AFL instances in parallel mode. +Although it is possible to run Eclipser alone, it is intended only for simple +testing and not for realistic fuzzing. ``` -$ dotnet build/Eclipser.dll fuzz \ - -p -v -t \ - -o --src <'arg'|'file'|'stdin'|'auto'> +$ AFL_DIR/afl-fuzz -i -o -M \ + -f -Q -- +$ AFL_DIR/afl-fuzz -i -o -S \ + -f -Q -- +$ dotnet ECLIPSER_DIR/build/Eclipser.dll \ + -t -i -s -o \ + -p --arg -f ``` -This command will fuzz the specified target program for the given amount of -time, and store the fuzzing outputs (i.e. test cases and crashes) into the -output directory. The last argument `--src ...` specifies target program's input -source to fuzz, and further explanation about this option is given below. +We note that the output directory for Eclipser should be placed under the +synchronization directory (e.g. `-s ../syncdir -o ../syncdir/eclipser-output`). +AFL will automatically create an output directory under the synchronization +directory, using its specified ID. This way, Eclipser and AFL will share test +cases with each other. To obtain the final result of the fuzzing, retrieve all +the test cases under `/*/queue/` and `/*/crashes/`. -- Fuzzing command line arguments of a target program +Similarly to AFL, Eclipser will fuzz the file input specified by `-f` option, and +fuzz the standard input when `-f` option is not provided. However, Eclipser does +not support `@@` syntax used by AFL. -By providing `--src arg` option to Eclipser, you can fuzz command-line argument -input of the target program. You can specify the number of arguments and the -maximum length of each argument string with `--maxarglen` option. +- Examples -For example, the following command will fuzz target program 'example.bin' by -mutating command line argument inputs. A generated test input can have up to -three argument strings (i.e. argc <= 3), and the length of the first argument -string is limited up to 8 bytes, while the second and third arguments are -confined to 4-bytes strings. - -``` -$ dotnet build/Eclipser.dll fuzz \ - -p example.bin -v 1 -t 10 -o outdir --src arg --maxarglen 8 4 4 -``` - -- Fuzzing a file input of a target program - -By providing `--src file` option to Eclipser, you can fuzz a file input of a -target program. You can specify the command line argument of target program with -`--initarg` option, and specify the input file name with `-f` option. - -For example, consider a target program that takes in input file via "--input" -option. Using the following command, you can fuzz the file input of this program -and limit the file input length up to 8 bytes. Currently we support only one -file input. - -``` -$ dotnet build/Eclipser.dll fuzz \ - -p example.bin -v 1 -t 10 -o outdir \ - --src file --initarg "--input foo" -f foo --maxfilelen 8 -``` - -You may also want to provide initial seed inputs for the fuzzing. You can use -`-i ` option to provide initial seed input files for the target -program. - -- Fuzzing the standard input of a target program - -By providing `--src stdin` to Eclipser, you can fuzz the standard input of a -target program. Currently, we assume a standard input to be a single string, and -do not consider cases where a sequence of string is provided as a standard input -stream of the target program. - -For example, the following command will fuzz target program 'example.bin' by -mutating its standard input. The length of standard input is confined up to 8 -bytes. - -``` -$ dotnet build/Eclipser.dll fuzz \ - -p example.bin -v 1 -t 10 -o outdir --src stdin --maxstdinlen 8 -``` - -- Fuzzing multiple input sources. - -Eclipser also supports a mode that automatically identifies and fuzz input -sources. When you provide `--src auto` option to Eclipser, it will first start -by fuzzing command line argument of the target program. Then, it will trace -system call invocations during the program execution, to identify the use of -standard input or file input. Once identified, Eclipser will automatically fuzz -these input sources as well. As in previous examples, you can specify the -maximum lengths with `--max*` options. - -``` -$ dotnet build/Eclipser.dll fuzz \ - -p example.bin -v 1 -t 10 -o outdir \ - --src auto --maxarglen 8 4 4 --maxfilelen 8 --maxstdinlen 8 -``` - -Note: Eclipser identifies a file input as an input source only if the file name -matches one of the argument string. This means if a target program reads in a -configuration file from a fixed path, this file will not be considered as a -input source to fuzz in `--src auto` mode. +You can find simple example programs and their fuzzing scripts in +[examples](./examples) directory. An example script to run Eclipser with AFL can +be found [here](examples/test_integerate.sh). Note that we create separate +working directories for each AFL instance and Eclipser in this script. This is +to prevent the instances from using the same input file path for fuzzing. - Other options for fuzzing -You can get the full list of Eclipser's options and their detailed descriptions -by running the following command. +You can get the full list of Eclipser's options and their descriptions by +running the following command. ``` -$ dotnet build/Eclipser.dll fuzz --help +$ dotnet build/Eclipser.dll --help ``` -# Test case decoding utility +# Eclipser v2.0 -Eclipser internally store test cases in its custom JSON format. For usability, -we provide a utility that decodes these JSON format test cases in a specified -directory. - -For example, suppose that you fuzzed a target program with the following -command. - -``` -$ dotnet build/Eclipser.dll fuzz -p example.bin -v 1 -t 10 -o outdir --src auto -``` - -Then, Eclipser will store generated test cases in `outdir/testcase`, and store -found crash inputs in `outdir/crash`. Now, to decode the test cases, you can run -the following command. Note the `decode` keyword is used in place of `fuzz`. - -``` -$ dotnet build/Eclipser.dll decode -i outdir/testcase -o decoded_testcase -``` - -Then, Eclipser will store the decoded raw string inputs in subdirectories, as -shown below. - -``` -$ ls decoded_testcase -decoded_args decoded_files decoded_paths decoded_stdins -$ xxd decoded_testcase/decoded_files/tc-0 -00000000: 4142 4242 4242 4242 4242 4242 4242 4242 ABBBBBBBBBBBBBBB -$ xxd decoded_testcase/decoded_files/tc-1 -00000000: 6142 4242 4242 4242 4242 4242 4242 4242 aBBBBBBBBBBBBBBB -(...) -``` +Originally, Eclipser had its own simplified random-based fuzzing module, instead +of relying on AFL. This was to support fuzzing multiple input sources (e.g. +command-line arguments, standard input, and file input) within a single fuzzer +run. We needed this feature for the comparison against KLEE on Coreutils +benchmark, which was one of the main experimental targets in our paper. -# Examples +However, as Eclipser is more often compared with other fuzzing tools, we abandon +this feature and focus on fuzzing a single input source, as most fuzzers do. We +also largely updated the command line interface of Eclipser accordingly. We note +that you can still checkout v1.0 code from our repository to reproduce the +Coreutils experiment result. -You can find simple example programs, along with testing scripts to fuzz these -programs, in [examples](./examples) directory. +By focusing on fuzzing a single input source, we can now use AFL to perform +random-based fuzzing. For this, from v2.0 Eclipser runs in parallel with AFL, as +described above. This way, we can benefit from various features offered by AFL, +such as source-based instrumentation, persistent mode, and deterministic mode. +Still, the core architecture of Eclipser remains the same: it complements +random-based fuzzing with our grey-box concolic testing technique. # Docker We also provide a Docker image to run the experiments in our paper, in [Eclipser-Artifact](https://github.com/SoftSec-KAIST/Eclipser-Artifact) -repository. +repository. Note that this image uses Eclipser v0.1, since the image was +built for the artifact evaluation of ICSE 2019. # Supported Architectures @@ -195,7 +125,7 @@ Eclipser. Please consider citing our paper (ICSE 2019): ```bibtex -@INPROCEEDINGS{choi:icse:2019, +@INPROCEEDINGS{choi:icse:2019, author = {Jaeseung Choi and Joonun Jang and Choongwoo Han and Sang Kil Cha}, title = {Grey-box Concolic Testing on Binary Code}, booktitle = {Proceedings of the International Conference on Software Engineering},