forked from Qortal/Brooklyn
506 lines
22 KiB
C++
506 lines
22 KiB
C++
//
|
|
// Copyright © 2017 Arm Ltd. All rights reserved.
|
|
// SPDX-License-Identifier: MIT
|
|
//
|
|
|
|
#include "../ImageTensorGenerator/ImageTensorGenerator.hpp"
|
|
#include "../InferenceTest.hpp"
|
|
#include "ModelAccuracyChecker.hpp"
|
|
#include "armnnDeserializer/IDeserializer.hpp"
|
|
|
|
#include <armnnUtils/Filesystem.hpp>
|
|
#include <armnnUtils/TContainer.hpp>
|
|
|
|
#include <cxxopts/cxxopts.hpp>
|
|
#include <map>
|
|
|
|
using namespace armnn::test;
|
|
|
|
/** Load image names and ground-truth labels from the image directory and the ground truth label file
|
|
*
|
|
* @pre \p validationLabelPath exists and is valid regular file
|
|
* @pre \p imageDirectoryPath exists and is valid directory
|
|
* @pre labels in validation file correspond to images which are in lexicographical order with the image name
|
|
* @pre image index starts at 1
|
|
* @pre \p begIndex and \p endIndex are end-inclusive
|
|
*
|
|
* @param[in] validationLabelPath Path to validation label file
|
|
* @param[in] imageDirectoryPath Path to directory containing validation images
|
|
* @param[in] begIndex Begin index of images to be loaded. Inclusive
|
|
* @param[in] endIndex End index of images to be loaded. Inclusive
|
|
* @param[in] excludelistPath Path to excludelist file
|
|
* @return A map mapping image file names to their corresponding ground-truth labels
|
|
*/
|
|
map<std::string, std::string> LoadValidationImageFilenamesAndLabels(const string& validationLabelPath,
|
|
const string& imageDirectoryPath,
|
|
size_t begIndex = 0,
|
|
size_t endIndex = 0,
|
|
const string& excludelistPath = "");
|
|
|
|
/** Load model output labels from file
|
|
*
|
|
* @pre \p modelOutputLabelsPath exists and is a regular file
|
|
*
|
|
* @param[in] modelOutputLabelsPath path to model output labels file
|
|
* @return A vector of labels, which in turn is described by a list of category names
|
|
*/
|
|
std::vector<armnnUtils::LabelCategoryNames> LoadModelOutputLabels(const std::string& modelOutputLabelsPath);
|
|
|
|
int main(int argc, char* argv[])
|
|
{
|
|
try
|
|
{
|
|
armnn::LogSeverity level = armnn::LogSeverity::Debug;
|
|
armnn::ConfigureLogging(true, true, level);
|
|
|
|
std::string modelPath;
|
|
std::string modelFormat;
|
|
std::vector<std::string> inputNames;
|
|
std::vector<std::string> outputNames;
|
|
std::string dataDir;
|
|
std::string modelOutputLabelsPath;
|
|
std::string validationLabelPath;
|
|
std::string inputLayout;
|
|
std::vector<armnn::BackendId> computeDevice;
|
|
std::string validationRange;
|
|
std::string excludelistPath;
|
|
|
|
const std::string backendsMessage = "Which device to run layers on by default. Possible choices: "
|
|
+ armnn::BackendRegistryInstance().GetBackendIdsAsString();
|
|
|
|
try
|
|
{
|
|
cxxopts::Options options("ModeAccuracyTool-Armnn","Options");
|
|
|
|
options.add_options()
|
|
("h,help", "Display help messages")
|
|
("m,model-path",
|
|
"Path to armnn format model file",
|
|
cxxopts::value<std::string>(modelPath))
|
|
("f,model-format",
|
|
"The model format. Supported values: tflite",
|
|
cxxopts::value<std::string>(modelFormat))
|
|
("i,input-name",
|
|
"Identifier of the input tensors in the network separated by comma with no space.",
|
|
cxxopts::value<std::vector<std::string>>(inputNames))
|
|
("o,output-name",
|
|
"Identifier of the output tensors in the network separated by comma with no space.",
|
|
cxxopts::value<std::vector<std::string>>(outputNames))
|
|
("d,data-dir",
|
|
"Path to directory containing the ImageNet test data",
|
|
cxxopts::value<std::string>(dataDir))
|
|
("p,model-output-labels",
|
|
"Path to model output labels file.",
|
|
cxxopts::value<std::string>(modelOutputLabelsPath))
|
|
("v,validation-labels-path",
|
|
"Path to ImageNet Validation Label file",
|
|
cxxopts::value<std::string>(validationLabelPath))
|
|
("l,data-layout",
|
|
"Data layout. Supported value: NHWC, NCHW. Default: NHWC",
|
|
cxxopts::value<std::string>(inputLayout)->default_value("NHWC"))
|
|
("c,compute",
|
|
backendsMessage.c_str(),
|
|
cxxopts::value<std::vector<armnn::BackendId>>(computeDevice)->default_value("CpuAcc,CpuRef"))
|
|
("r,validation-range",
|
|
"The range of the images to be evaluated. Specified in the form <begin index>:<end index>."
|
|
"The index starts at 1 and the range is inclusive."
|
|
"By default the evaluation will be performed on all images.",
|
|
cxxopts::value<std::string>(validationRange)->default_value("1:0"))
|
|
("e,excludelist-path",
|
|
"Path to a excludelist file where each line denotes the index of an image to be "
|
|
"excluded from evaluation.",
|
|
cxxopts::value<std::string>(excludelistPath)->default_value(""));
|
|
ARMNN_DEPRECATED_MSG_REMOVAL_DATE("This b,blacklist-path command is deprecated", "22.08")
|
|
("b,blacklist-path",
|
|
"Path to a blacklist file where each line denotes the index of an image to be "
|
|
"excluded from evaluation. This command will be deprecated in favor of: --excludelist-path ",
|
|
cxxopts::value<std::string>(excludelistPath)->default_value(""));
|
|
|
|
auto result = options.parse(argc, argv);
|
|
|
|
if (result.count("help") > 0)
|
|
{
|
|
std::cout << options.help() << std::endl;
|
|
return EXIT_FAILURE;
|
|
}
|
|
|
|
// Check for mandatory single options.
|
|
std::string mandatorySingleParameters[] = { "model-path", "model-format", "input-name", "output-name",
|
|
"data-dir", "model-output-labels", "validation-labels-path" };
|
|
for (auto param : mandatorySingleParameters)
|
|
{
|
|
if (result.count(param) != 1)
|
|
{
|
|
std::cerr << "Parameter \'--" << param << "\' is required but missing." << std::endl;
|
|
return EXIT_FAILURE;
|
|
}
|
|
}
|
|
}
|
|
catch (const cxxopts::OptionException& e)
|
|
{
|
|
std::cerr << e.what() << std::endl << std::endl;
|
|
return EXIT_FAILURE;
|
|
}
|
|
catch (const std::exception& e)
|
|
{
|
|
ARMNN_ASSERT_MSG(false, "Caught unexpected exception");
|
|
std::cerr << "Fatal internal error: " << e.what() << std::endl;
|
|
return EXIT_FAILURE;
|
|
}
|
|
|
|
// Check if the requested backend are all valid
|
|
std::string invalidBackends;
|
|
if (!CheckRequestedBackendsAreValid(computeDevice, armnn::Optional<std::string&>(invalidBackends)))
|
|
{
|
|
ARMNN_LOG(fatal) << "The list of preferred devices contains invalid backend IDs: "
|
|
<< invalidBackends;
|
|
return EXIT_FAILURE;
|
|
}
|
|
armnn::Status status;
|
|
|
|
// Create runtime
|
|
armnn::IRuntime::CreationOptions options;
|
|
armnn::IRuntimePtr runtime(armnn::IRuntime::Create(options));
|
|
std::ifstream file(modelPath);
|
|
|
|
// Create Parser
|
|
using IParser = armnnDeserializer::IDeserializer;
|
|
auto armnnparser(IParser::Create());
|
|
|
|
// Create a network
|
|
armnn::INetworkPtr network = armnnparser->CreateNetworkFromBinary(file);
|
|
|
|
// Optimizes the network.
|
|
armnn::IOptimizedNetworkPtr optimizedNet(nullptr, nullptr);
|
|
try
|
|
{
|
|
optimizedNet = armnn::Optimize(*network, computeDevice, runtime->GetDeviceSpec());
|
|
}
|
|
catch (const armnn::Exception& e)
|
|
{
|
|
std::stringstream message;
|
|
message << "armnn::Exception (" << e.what() << ") caught from optimize.";
|
|
ARMNN_LOG(fatal) << message.str();
|
|
return EXIT_FAILURE;
|
|
}
|
|
|
|
// Loads the network into the runtime.
|
|
armnn::NetworkId networkId;
|
|
status = runtime->LoadNetwork(networkId, std::move(optimizedNet));
|
|
if (status == armnn::Status::Failure)
|
|
{
|
|
ARMNN_LOG(fatal) << "armnn::IRuntime: Failed to load network";
|
|
return EXIT_FAILURE;
|
|
}
|
|
|
|
// Set up Network
|
|
using BindingPointInfo = InferenceModelInternal::BindingPointInfo;
|
|
|
|
// Handle inputNames and outputNames, there can be multiple.
|
|
std::vector<BindingPointInfo> inputBindings;
|
|
for(auto& input: inputNames)
|
|
{
|
|
const armnnDeserializer::BindingPointInfo&
|
|
inputBindingInfo = armnnparser->GetNetworkInputBindingInfo(0, input);
|
|
|
|
std::pair<armnn::LayerBindingId, armnn::TensorInfo>
|
|
m_InputBindingInfo(inputBindingInfo.m_BindingId, inputBindingInfo.m_TensorInfo);
|
|
inputBindings.push_back(m_InputBindingInfo);
|
|
}
|
|
|
|
std::vector<BindingPointInfo> outputBindings;
|
|
for(auto& output: outputNames)
|
|
{
|
|
const armnnDeserializer::BindingPointInfo&
|
|
outputBindingInfo = armnnparser->GetNetworkOutputBindingInfo(0, output);
|
|
|
|
std::pair<armnn::LayerBindingId, armnn::TensorInfo>
|
|
m_OutputBindingInfo(outputBindingInfo.m_BindingId, outputBindingInfo.m_TensorInfo);
|
|
outputBindings.push_back(m_OutputBindingInfo);
|
|
}
|
|
|
|
// Load model output labels
|
|
if (modelOutputLabelsPath.empty() || !fs::exists(modelOutputLabelsPath) ||
|
|
!fs::is_regular_file(modelOutputLabelsPath))
|
|
{
|
|
ARMNN_LOG(fatal) << "Invalid model output labels path at " << modelOutputLabelsPath;
|
|
}
|
|
const std::vector<armnnUtils::LabelCategoryNames> modelOutputLabels =
|
|
LoadModelOutputLabels(modelOutputLabelsPath);
|
|
|
|
// Parse begin and end image indices
|
|
std::vector<std::string> imageIndexStrs = armnnUtils::SplitBy(validationRange, ":");
|
|
size_t imageBegIndex;
|
|
size_t imageEndIndex;
|
|
if (imageIndexStrs.size() != 2)
|
|
{
|
|
ARMNN_LOG(fatal) << "Invalid validation range specification: Invalid format " << validationRange;
|
|
return EXIT_FAILURE;
|
|
}
|
|
try
|
|
{
|
|
imageBegIndex = std::stoul(imageIndexStrs[0]);
|
|
imageEndIndex = std::stoul(imageIndexStrs[1]);
|
|
}
|
|
catch (const std::exception& e)
|
|
{
|
|
ARMNN_LOG(fatal) << "Invalid validation range specification: " << validationRange;
|
|
return EXIT_FAILURE;
|
|
}
|
|
|
|
// Validate excludelist file if it's specified
|
|
if (!excludelistPath.empty() &&
|
|
!(fs::exists(excludelistPath) && fs::is_regular_file(excludelistPath)))
|
|
{
|
|
ARMNN_LOG(fatal) << "Invalid path to excludelist file at " << excludelistPath;
|
|
return EXIT_FAILURE;
|
|
}
|
|
|
|
fs::path pathToDataDir(dataDir);
|
|
const map<std::string, std::string> imageNameToLabel = LoadValidationImageFilenamesAndLabels(
|
|
validationLabelPath, pathToDataDir.string(), imageBegIndex, imageEndIndex, excludelistPath);
|
|
armnnUtils::ModelAccuracyChecker checker(imageNameToLabel, modelOutputLabels);
|
|
|
|
if (ValidateDirectory(dataDir))
|
|
{
|
|
InferenceModel<armnnDeserializer::IDeserializer, float>::Params params;
|
|
|
|
params.m_ModelPath = modelPath;
|
|
params.m_IsModelBinary = true;
|
|
params.m_ComputeDevices = computeDevice;
|
|
// Insert inputNames and outputNames into params vector
|
|
params.m_InputBindings.insert(std::end(params.m_InputBindings),
|
|
std::begin(inputNames),
|
|
std::end(inputNames));
|
|
params.m_OutputBindings.insert(std::end(params.m_OutputBindings),
|
|
std::begin(outputNames),
|
|
std::end(outputNames));
|
|
|
|
using TParser = armnnDeserializer::IDeserializer;
|
|
// If dynamicBackends is empty it will be disabled by default.
|
|
InferenceModel<TParser, float> model(params, false, "");
|
|
|
|
// Get input tensor information
|
|
const armnn::TensorInfo& inputTensorInfo = model.GetInputBindingInfo().second;
|
|
const armnn::TensorShape& inputTensorShape = inputTensorInfo.GetShape();
|
|
const armnn::DataType& inputTensorDataType = inputTensorInfo.GetDataType();
|
|
armnn::DataLayout inputTensorDataLayout;
|
|
if (inputLayout == "NCHW")
|
|
{
|
|
inputTensorDataLayout = armnn::DataLayout::NCHW;
|
|
}
|
|
else if (inputLayout == "NHWC")
|
|
{
|
|
inputTensorDataLayout = armnn::DataLayout::NHWC;
|
|
}
|
|
else
|
|
{
|
|
ARMNN_LOG(fatal) << "Invalid Data layout: " << inputLayout;
|
|
return EXIT_FAILURE;
|
|
}
|
|
const unsigned int inputTensorWidth =
|
|
inputTensorDataLayout == armnn::DataLayout::NCHW ? inputTensorShape[3] : inputTensorShape[2];
|
|
const unsigned int inputTensorHeight =
|
|
inputTensorDataLayout == armnn::DataLayout::NCHW ? inputTensorShape[2] : inputTensorShape[1];
|
|
// Get output tensor info
|
|
const unsigned int outputNumElements = model.GetOutputSize();
|
|
// Check output tensor shape is valid
|
|
if (modelOutputLabels.size() != outputNumElements)
|
|
{
|
|
ARMNN_LOG(fatal) << "Number of output elements: " << outputNumElements
|
|
<< " , mismatches the number of output labels: " << modelOutputLabels.size();
|
|
return EXIT_FAILURE;
|
|
}
|
|
|
|
const unsigned int batchSize = 1;
|
|
// Get normalisation parameters
|
|
SupportedFrontend modelFrontend;
|
|
if (modelFormat == "tflite")
|
|
{
|
|
modelFrontend = SupportedFrontend::TFLite;
|
|
}
|
|
else
|
|
{
|
|
ARMNN_LOG(fatal) << "Unsupported frontend: " << modelFormat;
|
|
return EXIT_FAILURE;
|
|
}
|
|
const NormalizationParameters& normParams = GetNormalizationParameters(modelFrontend, inputTensorDataType);
|
|
for (const auto& imageEntry : imageNameToLabel)
|
|
{
|
|
const std::string imageName = imageEntry.first;
|
|
std::cout << "Processing image: " << imageName << "\n";
|
|
|
|
vector<armnnUtils::TContainer> inputDataContainers;
|
|
vector<armnnUtils::TContainer> outputDataContainers;
|
|
|
|
auto imagePath = pathToDataDir / fs::path(imageName);
|
|
switch (inputTensorDataType)
|
|
{
|
|
case armnn::DataType::Signed32:
|
|
inputDataContainers.push_back(
|
|
PrepareImageTensor<int>(imagePath.string(),
|
|
inputTensorWidth, inputTensorHeight,
|
|
normParams,
|
|
batchSize,
|
|
inputTensorDataLayout));
|
|
outputDataContainers = { vector<int>(outputNumElements) };
|
|
break;
|
|
case armnn::DataType::QAsymmU8:
|
|
inputDataContainers.push_back(
|
|
PrepareImageTensor<uint8_t>(imagePath.string(),
|
|
inputTensorWidth, inputTensorHeight,
|
|
normParams,
|
|
batchSize,
|
|
inputTensorDataLayout));
|
|
outputDataContainers = { vector<uint8_t>(outputNumElements) };
|
|
break;
|
|
case armnn::DataType::Float32:
|
|
default:
|
|
inputDataContainers.push_back(
|
|
PrepareImageTensor<float>(imagePath.string(),
|
|
inputTensorWidth, inputTensorHeight,
|
|
normParams,
|
|
batchSize,
|
|
inputTensorDataLayout));
|
|
outputDataContainers = { vector<float>(outputNumElements) };
|
|
break;
|
|
}
|
|
|
|
status = runtime->EnqueueWorkload(networkId,
|
|
armnnUtils::MakeInputTensors(inputBindings, inputDataContainers),
|
|
armnnUtils::MakeOutputTensors(outputBindings, outputDataContainers));
|
|
|
|
if (status == armnn::Status::Failure)
|
|
{
|
|
ARMNN_LOG(fatal) << "armnn::IRuntime: Failed to enqueue workload for image: " << imageName;
|
|
}
|
|
|
|
checker.AddImageResult<armnnUtils::TContainer>(imageName, outputDataContainers);
|
|
}
|
|
}
|
|
else
|
|
{
|
|
return EXIT_SUCCESS;
|
|
}
|
|
|
|
for(unsigned int i = 1; i <= 5; ++i)
|
|
{
|
|
std::cout << "Top " << i << " Accuracy: " << checker.GetAccuracy(i) << "%" << "\n";
|
|
}
|
|
|
|
ARMNN_LOG(info) << "Accuracy Tool ran successfully!";
|
|
return EXIT_SUCCESS;
|
|
}
|
|
catch (const armnn::Exception& e)
|
|
{
|
|
// Coverity fix: BOOST_LOG_TRIVIAL (typically used to report errors) may throw an
|
|
// exception of type std::length_error.
|
|
// Using stderr instead in this context as there is no point in nesting try-catch blocks here.
|
|
std::cerr << "Armnn Error: " << e.what() << std::endl;
|
|
return EXIT_FAILURE;
|
|
}
|
|
catch (const std::exception& e)
|
|
{
|
|
// Coverity fix: various boost exceptions can be thrown by methods called by this test.
|
|
std::cerr << "WARNING: ModelAccuracyTool-Armnn: An error has occurred when running the "
|
|
"Accuracy Tool: " << e.what() << std::endl;
|
|
return EXIT_FAILURE;
|
|
}
|
|
}
|
|
|
|
map<std::string, std::string> LoadValidationImageFilenamesAndLabels(const string& validationLabelPath,
|
|
const string& imageDirectoryPath,
|
|
size_t begIndex,
|
|
size_t endIndex,
|
|
const string& excludelistPath)
|
|
{
|
|
// Populate imageFilenames with names of all .JPEG, .PNG images
|
|
std::vector<std::string> imageFilenames;
|
|
for (const auto& imageEntry : fs::directory_iterator(fs::path(imageDirectoryPath)))
|
|
{
|
|
fs::path imagePath = imageEntry.path();
|
|
|
|
// Get extension and convert to uppercase
|
|
std::string imageExtension = imagePath.extension().string();
|
|
std::transform(imageExtension.begin(), imageExtension.end(), imageExtension.begin(), ::toupper);
|
|
|
|
if (fs::is_regular_file(imagePath) && (imageExtension == ".JPEG" || imageExtension == ".PNG"))
|
|
{
|
|
imageFilenames.push_back(imagePath.filename().string());
|
|
}
|
|
}
|
|
if (imageFilenames.empty())
|
|
{
|
|
throw armnn::Exception("No image file (JPEG, PNG) found at " + imageDirectoryPath);
|
|
}
|
|
|
|
// Sort the image filenames lexicographically
|
|
std::sort(imageFilenames.begin(), imageFilenames.end());
|
|
|
|
std::cout << imageFilenames.size() << " images found at " << imageDirectoryPath << std::endl;
|
|
|
|
// Get default end index
|
|
if (begIndex < 1 || endIndex > imageFilenames.size())
|
|
{
|
|
throw armnn::Exception("Invalid image index range");
|
|
}
|
|
endIndex = endIndex == 0 ? imageFilenames.size() : endIndex;
|
|
if (begIndex > endIndex)
|
|
{
|
|
throw armnn::Exception("Invalid image index range");
|
|
}
|
|
|
|
// Load excludelist if there is one
|
|
std::vector<unsigned int> excludelist;
|
|
if (!excludelistPath.empty())
|
|
{
|
|
std::ifstream excludelistFile(excludelistPath);
|
|
unsigned int index;
|
|
while (excludelistFile >> index)
|
|
{
|
|
excludelist.push_back(index);
|
|
}
|
|
}
|
|
|
|
// Load ground truth labels and pair them with corresponding image names
|
|
std::string classification;
|
|
map<std::string, std::string> imageNameToLabel;
|
|
ifstream infile(validationLabelPath);
|
|
size_t imageIndex = begIndex;
|
|
size_t excludelistIndexCount = 0;
|
|
while (std::getline(infile, classification))
|
|
{
|
|
if (imageIndex > endIndex)
|
|
{
|
|
break;
|
|
}
|
|
// If current imageIndex is included in excludelist, skip the current image
|
|
if (excludelistIndexCount < excludelist.size() && imageIndex == excludelist[excludelistIndexCount])
|
|
{
|
|
++imageIndex;
|
|
++excludelistIndexCount;
|
|
continue;
|
|
}
|
|
imageNameToLabel.insert(std::pair<std::string, std::string>(imageFilenames[imageIndex - 1], classification));
|
|
++imageIndex;
|
|
}
|
|
std::cout << excludelistIndexCount << " images in excludelist" << std::endl;
|
|
std::cout << imageIndex - begIndex - excludelistIndexCount << " images to be loaded" << std::endl;
|
|
return imageNameToLabel;
|
|
}
|
|
|
|
std::vector<armnnUtils::LabelCategoryNames> LoadModelOutputLabels(const std::string& modelOutputLabelsPath)
|
|
{
|
|
std::vector<armnnUtils::LabelCategoryNames> modelOutputLabels;
|
|
ifstream modelOutputLablesFile(modelOutputLabelsPath);
|
|
std::string line;
|
|
while (std::getline(modelOutputLablesFile, line))
|
|
{
|
|
armnnUtils::LabelCategoryNames tokens = armnnUtils::SplitBy(line, ":");
|
|
armnnUtils::LabelCategoryNames predictionCategoryNames = armnnUtils::SplitBy(tokens.back(), ",");
|
|
std::transform(predictionCategoryNames.begin(), predictionCategoryNames.end(), predictionCategoryNames.begin(),
|
|
[](const std::string& category) { return armnnUtils::Strip(category); });
|
|
modelOutputLabels.push_back(predictionCategoryNames);
|
|
}
|
|
return modelOutputLabels;
|
|
} |