diff --git a/Alignment/README.md b/Alignment/README.md new file mode 100644 index 0000000..01c3245 --- /dev/null +++ b/Alignment/README.md @@ -0,0 +1,15 @@ +## Aligner for scanned .sens, .ply files +========================================= + +Transforms scans to z-up alignment for scans and tries to align walls to x-y planes. + +### Installation. +This code was developed under VS2013. + +Requirements: +- our research library mLib, a git submodule in ../external/mLib +- mLib external libraries can be downloaded [here](https://www.dropbox.com/s/fve3uen5mzonidx/mLibExternal.zip?dl=0) + + +To run: +`alignment.exe [path to directory of scan to align]` \ No newline at end of file diff --git a/Alignment/alignment.sln b/Alignment/alignment.sln new file mode 100644 index 0000000..ef6367d --- /dev/null +++ b/Alignment/alignment.sln @@ -0,0 +1,22 @@ + +Microsoft Visual Studio Solution File, Format Version 12.00 +# Visual Studio 2013 +VisualStudioVersion = 12.0.31101.0 +MinimumVisualStudioVersion = 10.0.40219.1 +Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "alignment", "alignment.vcxproj", "{EB0D04B5-579C-4BF9-8FC3-58DFB235231F}" +EndProject +Global + GlobalSection(SolutionConfigurationPlatforms) = preSolution + Debug|x64 = Debug|x64 + Release|x64 = Release|x64 + EndGlobalSection + GlobalSection(ProjectConfigurationPlatforms) = postSolution + {EB0D04B5-579C-4BF9-8FC3-58DFB235231F}.Debug|x64.ActiveCfg = Debug|x64 + {EB0D04B5-579C-4BF9-8FC3-58DFB235231F}.Debug|x64.Build.0 = Debug|x64 + {EB0D04B5-579C-4BF9-8FC3-58DFB235231F}.Release|x64.ActiveCfg = Release|x64 + {EB0D04B5-579C-4BF9-8FC3-58DFB235231F}.Release|x64.Build.0 = Release|x64 + EndGlobalSection + GlobalSection(SolutionProperties) = preSolution + HideSolutionNode = FALSE + EndGlobalSection +EndGlobal diff --git a/Alignment/alignment.vcxproj b/Alignment/alignment.vcxproj new file mode 100644 index 0000000..02c79cc --- /dev/null +++ b/Alignment/alignment.vcxproj @@ -0,0 +1,105 @@ + + + + + Debug + x64 + + + Release + x64 + + + + + + + + + + + + + + + true + true + + + Create + Create + + + + {EB0D04B5-579C-4BF9-8FC3-58DFB235231F} + alignment + + + + Application + true + v120 + MultiByte + + + Application + false + v120 + true + MultiByte + + + + + + + + + + + + + ../external/mLib/include;../external/mLibExternal/include/;./src/;$(IncludePath) + ..\external\mLibExternal\libsWindows\lib64;$(LibraryPath) + + + ../external/mLib/include;../external/mLibExternal/include/;./src/;$(IncludePath) + ..\external\mLibExternal\libsWindows\lib64;$(LibraryPath) + + + + Level3 + Disabled + true + _CRT_SECURE_NO_WARNINGS;_DEBUG;%(PreprocessorDefinitions) + Use + -Zm400 %(AdditionalOptions) + + + true + freeimage.lib;%(AdditionalDependencies) + + + + + Level2 + MaxSpeed + true + true + true + _CRT_SECURE_NO_WARNINGS;%(PreprocessorDefinitions) + true + Use + -Zm400 %(AdditionalOptions) + + + true + true + true + freeimage.lib;%(AdditionalDependencies) + + + + + + \ No newline at end of file diff --git a/Alignment/alignment.vcxproj.filters b/Alignment/alignment.vcxproj.filters new file mode 100644 index 0000000..2c9c538 --- /dev/null +++ b/Alignment/alignment.vcxproj.filters @@ -0,0 +1,17 @@ + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/Alignment/src/alignment.h b/Alignment/src/alignment.h new file mode 100644 index 0000000..588c020 --- /dev/null +++ b/Alignment/src/alignment.h @@ -0,0 +1,366 @@ + +#pragma once + +#include "stdafx.h" + +#include "processedFile.h" +#include "planeExtract.h" + +class Alignment { +public: + + static MeshDataf makeNormalMesh(const vec3f& p0, const vec3f& p1, const vec4f& color = vec4f(0.8f, 0.2f, 0.2f, 1.0f)) + { + TriMeshf normalMesh0 = Shapesf::sphere(0.05f, p0, 10, 10, color); + TriMeshf normalMesh1 = Shapesf::cylinder(p0, p1, 0.025f, 10, 10, color); + MeshDataf nMesh = normalMesh0.computeMeshData(); + nMesh.merge(normalMesh1.computeMeshData()); + return nMesh; + } + + + + static vec3f upVectorFromViews(const SensorData& sd) { + vec3f v(0.0f, 0.0f, 0.0f); + for (size_t i = 0; i < sd.m_frames.size(); i++) { + const mat4f& t = sd.m_frames[i].getCameraToWorld(); + if (sd.m_frames[i].getCameraToWorld()(0, 0) == -std::numeric_limits::infinity()) continue; + + const vec3f cameraUp(0.0f, -1.0f, 0.0f); + const vec3f worldUp = (t.getRotation() * cameraUp).getNormalized(); + v += worldUp; + } + v /= (float)sd.m_frames.size(); + return v.getNormalized(); + } + + static bool hasAccel(const SensorData& sd, unsigned int numThresh = 10) { + unsigned int numValidAccel = 0; + for (const SensorData::IMUFrame& f : sd.m_IMUFrames) { + if (f.acceleration != vec3d(0.0)) numValidAccel++; + } + + if (numValidAccel > numThresh) return true; + else return false; + } + + static vec3f upVectorFromAccel(const SensorData& sd) + { + if (sd.m_IMUFrames.size() == 0) throw MLIB_EXCEPTION("no imu data found"); + + vec3f v(0.0f, 0.0f, 0.0f); + for (size_t i = 0; i < sd.m_frames.size(); i++) { + const SensorData::IMUFrame& f = sd.findClosestIMUFrame(i); + + if (sd.m_frames[i].getCameraToWorld()(0, 0) == -std::numeric_limits::infinity()) continue; + + if (f.acceleration == vec3d::origin) { + std::cout << "invalid IMU acceleration data entry at " << i << "-th frame" << std::endl; + continue; + } + + vec3f cameraUp = -vec3f((float)f.acceleration.x, (float)f.acceleration.y, (float)f.acceleration.z).getNormalized(); + const mat4f& t = sd.m_frames[i].getCameraToWorld(); + const vec3f worldUp = (t.getRotation() * cameraUp).getNormalized(); + v += worldUp; + } + + return v.getNormalized(); + } + + static bool hasGravity(const SensorData& sd, unsigned int numThresh = 10) + { + unsigned int numValidGravity = 0; + for (const SensorData::IMUFrame& f : sd.m_IMUFrames) { + if (f.gravity != vec3d(0.0)) numValidGravity++; + } + + if (numValidGravity > numThresh) return true; + else return false; + } + + static vec3f upVectorFromGravity(const SensorData& sd) + { + if (sd.m_IMUFrames.size() == 0) throw MLIB_EXCEPTION("no imu data found"); + + vec3f v(0.0f, 0.0f, 0.0f); + for (size_t i = 0; i < sd.m_frames.size(); i++) { + const SensorData::IMUFrame& f = sd.findClosestIMUFrame(i); + if (sd.m_frames[i].getCameraToWorld()(0, 0) == -std::numeric_limits::infinity()) continue; + + if (f.gravity == vec3d::origin) { + std::cout << "invalid IMU gravity data entry at " << i << "-th frame" << std::endl; + continue; + } + + vec3f cameraUp = vec3f((float)f.gravity.x, (float)f.gravity.y, (float)f.gravity.z).getNormalized(); + + cameraUp = vec3f(cameraUp.y, cameraUp.x, cameraUp.z); + + const mat4f& t = sd.m_frames[i].getCameraToWorld(); + const vec3f worldUp = (t.getRotation() * cameraUp).getNormalized(); + v += worldUp; + } + v /= (float)sd.m_frames.size(); + + return v.getNormalized(); + } + + static void writeTrajectoryFile(const std::string& outFile, const SensorData& sd) { + + std::vector trajecotry; + for (size_t i = 0; i < sd.m_frames.size(); i++) { + trajecotry.push_back(sd.m_frames[i].getCameraToWorld()); + } + + BinaryDataStreamFile outStream(outFile, true); + outStream << trajecotry; + outStream.close(); + } + + static std::vector readTrajectoryFile(const std::string& inFile) { + BinaryDataStreamFile in(inFile, false); + std::vector trajectory; + in >> trajectory; + return trajectory; + } + + static void removeInvalidIMUFrames(SensorData& sd) { + const bool checkTimeStamp = true; //at the moment only remove invalid time frames + const bool checkGravity = false; + const bool checkAccel = false; + + unsigned int removedInvalidFrames = 0; + for (std::vector::iterator iter = sd.m_IMUFrames.begin(); iter != sd.m_IMUFrames.end();) { + bool rem = false; + if (checkTimeStamp && iter->timeStamp == 0) rem = true; + if (checkGravity && iter->gravity == vec3d::origin) rem = true; + if (checkAccel && iter->gravity == vec3d::origin) rem = true; + + if (rem == true) { + iter = sd.m_IMUFrames.erase(iter); + removedInvalidFrames++; + } + else { + iter++; + } + } + + if (removedInvalidFrames > 0) { + std::cout << "removed " << removedInvalidFrames << " invalid IMUFrames" << std::endl; + } + } + + static void alignScan(const std::string& path, bool forceRealign = false) { + + const std::string processedFile(path + "/" + "processed.txt"); + if (!util::fileExists(processedFile)) { + std::cout << "no reconstruction available for " << path << "\n\t -> skipping folder" << std::endl; + return; + } + ParameterFile parameterFile(processedFile); + ProcessedFile pf; pf.readMembers(parameterFile); + if (!pf.valid) { + std::cout << "reconstruction was invalid for " << path << "\n\t -> skipping folder" << std::endl; + return; + } + if (pf.aligned && !forceRealign) { + std::cout << "reconstruction is already aligned " << path << "\n\t -> skipping folder" << std::endl; + return; + } + + Directory dir(path); + const std::vector tmp = ml::util::split(util::replace(path, "\\", "/"), "/"); //we assume forward slashes + const std::string base = tmp.back(); + const std::string baseFile = path + "/" + base; + + const std::string sensFile = path + "/" + base + ".sens"; + const std::string plyFile = path + "/" + base + ".ply"; + const std::string trajFile = util::replace(sensFile, ".sens", ".traj"); + + + mat4f transform = mat4f::identity(); + + SensorData sd(sensFile); + removeInvalidIMUFrames(sd); + + if (sd.m_frames.size() == 0) throw MLIB_EXCEPTION("no frames found in the sensor file"); + + if (sd.m_frames[0].getCameraToWorld() != mat4f::identity()) { + std::cout << "already found a previous alignment -> reverting to original" << std::endl; + + if (sd.m_frames[0].getCameraToWorld()(0, 0) == -std::numeric_limits::infinity()) { + std::cout << "error can't revert due to an invalid transform in the first frame" << std::endl; + std::cout << "\tskipping folder " << std::endl; + return; + } + mat4f inverse = sd.m_frames[0].getCameraToWorld().getInverse(); + sd.applyTransform(inverse); + + //if we have a multiple ply files (e.g., if VH was already run): + Directory dir(path); + std::vector plyFiles = dir.getFilesWithSuffix(".ply"); + for (const std::string& plyFile : plyFiles) { + MeshDataf md = MeshIOf::loadFromFile(dir.getPath() + "/" + plyFile); + md.applyTransform(inverse); + MeshIOf::saveToFile(dir.getPath() + "/" + plyFile, md); + } + } + + MeshDataf md = MeshIOf::loadFromFile(plyFile); + md.mergeCloseVertices(0.0005f, true); + md.removeIsolatedPieces(5000); + + //compute approx up vector from camera views or gravity (if available) + if (true) { + vec3f upEstimateView = upVectorFromViews(sd); + vec3f upEstimate = upEstimateView; + if (hasGravity(sd)) { //try to use gravity if possible + vec3f upEstimateGrav = upVectorFromGravity(sd); + upEstimate = upEstimateGrav; + } + + vec3f x = upEstimate ^ vec3f(upEstimate.y, -upEstimate.z, upEstimate.x); x.normalize(); + vec3f y = upEstimate ^ x; y.normalize(); + vec3f z = upEstimate; + mat4f mat = mat4f(x, y, z); + md.applyTransform(mat); + transform = mat * transform; + } + + md.computeVertexNormals(); + + //attempting to find a ground plane (and align to it) + if (true) { + PlaneExtract pe(md); + pe.cluster(); + pe.removeSmallClusters(); + pe.removeNonBoundingClusters(0.1f, 100); //the + const auto& clusters = pe.getClusters(); + + bool foundHorizontalPlane = false; + const size_t maxClusters = clusters.size(); + size_t i = 0; + for (const Cluster& c : clusters) { + if ((c.m_rep.plane.getNormal() | vec3f(0.0f, 0.0f, 1.0f)) > 0.8f) { + mat4f mat = c.reComputeNormalAlignment(); + md.applyTransform(mat); + transform = mat * transform; + foundHorizontalPlane = true; + break; + } + i++; + if (i >= maxClusters) break; + } + if (!foundHorizontalPlane) std::cout << "could not find a horizontal plane" << std::endl; + + //final alignment with xy plane (translation) + if (true) { + BoundingBox3f bb = md.computeBoundingBox(); + mat4f mat = mat4f::translation(-bb.getMinZ()); + transform = mat * transform; + md.applyTransform(mat); + + bb = md.computeBoundingBox(); + mat = mat4f::translation(-vec3f(bb.getCenter().x, bb.getCenter().y, 0.0f)); + md.applyTransform(mat); + transform = mat * transform; + } + + //attempting to find a vertical plane to align with the x and y axis + if (true) { + if (true) { + //first find the bounding box using cgal + OrientedBoundingBox3f obb = CGALWrapperf::computeOrientedBoundingBox(md.m_Vertices, CGALWrapperf::CONSTRAIN_Z); + mat4f mat = mat4f(obb.getAxisX().getNormalized(), obb.getAxisY().getNormalized(), obb.getAxisZ().getNormalized()); + md.applyTransform(mat); + transform = mat * transform; + } + + //make sure x and y are in the positive quadrant + if (true) { + BoundingBox3f bb = md.computeBoundingBox(); + mat4f mat = mat4f::translation(-vec3f(bb.getMin().x, bb.getMin().y, 0.0f)); + md.applyTransform(mat); + transform = mat * transform; + } + } + } + + md.m_Normals.clear(); + + { + //if we have a multiple ply files (e.g., if VH was already run): + Directory dir(path); + std::vector plyFiles = dir.getFilesWithSuffix(".ply"); + for (const std::string& plyFile : plyFiles) { + MeshDataf mesh = MeshIOf::loadFromFile(dir.getPath() + "/" + plyFile); + mesh.applyTransform(transform); + MeshIOf::saveToFile(dir.getPath() + "/" + plyFile, mesh); + } + + + sd.applyTransform(transform); + sd.saveToFile(sensFile); + pf.aligned = true; //it's now aligned + pf.saveToFile(processedFile); + } + } + + static mat4f readTransformFromAln(const std::string& filename) + { + std::ifstream s(filename); + if (!s.is_open()) throw MLIB_EXCEPTION("failed to open file " + filename + " for read"); + std::string tmp; + std::getline(s, tmp); std::getline(s, tmp); std::getline(s, tmp);//ignore header lines + mat4f m; + for (unsigned int i = 0; i < 16; i++) s >> m[i]; + s.close(); + return m; + } + + static void alignScanFromAlnFile(const std::string& path) { + + const std::string processedFile(path + "/" + "processed.txt"); + if (!util::fileExists(processedFile)) { + std::cout << "no reconstruction available for " << path << "\n\t -> skipping folder" << std::endl; + return; + } + ParameterFile parameterFile(processedFile); + ProcessedFile pf; pf.readMembers(parameterFile); + if (!pf.valid) { + std::cout << "reconstruction was invalid for " << path << "\n\t -> skipping folder" << std::endl; + return; + } + + Directory dir(path); + const std::vector tmp = ml::util::split(util::replace(path, "\\", "/"), "/"); //we assume forward slashes + const std::string base = tmp.back(); + const std::string baseFile = path + "/" + base; + + const std::string alnFile = path + "/alignment.aln"; + const std::string sensFile = path + "/" + base + ".sens"; + std::vector plyFiles = dir.getFilesWithSuffix(".ply"); + + const mat4f transform = readTransformFromAln(alnFile); + + SensorData sd(sensFile); + if (sd.m_frames.size() == 0) throw MLIB_EXCEPTION("no frames found in the sensor file"); + sd.applyTransform(transform); + + { //save out transformed + for (const std::string& plyFile : plyFiles) { + MeshDataf mesh = MeshIOf::loadFromFile(dir.getPath() + "/" + plyFile); + mesh.applyTransform(transform); + MeshIOf::saveToFile(dir.getPath() + "/" + plyFile, mesh); + } + sd.applyTransform(transform); + sd.saveToFile(sensFile); + pf.aligned = true; //it's now aligned + pf.saveToFile(processedFile); + } + } + + +private: +}; \ No newline at end of file diff --git a/Alignment/src/globalAppState.h b/Alignment/src/globalAppState.h new file mode 100644 index 0000000..1ff697b --- /dev/null +++ b/Alignment/src/globalAppState.h @@ -0,0 +1,72 @@ +#pragma once + +#ifndef _GLOBAL_APP_STATE_H_ +#define _GLOBAL_APP_STATE_H_ + +#include "mLibInclude.h" + +#define X_GLOBAL_APP_STATE_FIELDS \ + X(std::string, s_inputTestFile) \ + X(std::string, s_sourceDirectory) \ + X(std::vector, s_methods) + + +#ifndef VAR_NAME +#define VAR_NAME(x) #x +#endif + +#define checkSizeArray(a, d)( (((sizeof a)/(sizeof a[0])) >= d)) + +class GlobalAppState +{ +public: + +#define X(type, name) type name; + X_GLOBAL_APP_STATE_FIELDS +#undef X + + //! sets the parameter file and reads + void readMembers(const ParameterFile& parameterFile) { + m_ParameterFile = parameterFile; + readMembers(); + } + + //! reads all the members from the given parameter file (could be called for reloading) + void readMembers() { +#define X(type, name) \ + if (!m_ParameterFile.readParameter(std::string(#name), name)) {MLIB_WARNING(std::string(#name).append(" ").append("uninitialized")); name = type();} + X_GLOBAL_APP_STATE_FIELDS +#undef X + m_bIsInitialized = true; + } + + void print() const { +#define X(type, name) \ + std::cout << #name " = " << name << std::endl; + X_GLOBAL_APP_STATE_FIELDS +#undef X + } + + static GlobalAppState& get() { + static GlobalAppState s; + return s; + } + + + //! constructor + GlobalAppState() { + m_bIsInitialized = false; + } + + //! destructor + ~GlobalAppState() { + } + + + +private: + bool m_bIsInitialized; + ParameterFile m_ParameterFile; +}; + +#endif \ No newline at end of file diff --git a/Alignment/src/mLibInclude.h b/Alignment/src/mLibInclude.h new file mode 100644 index 0000000..4ea883d --- /dev/null +++ b/Alignment/src/mLibInclude.h @@ -0,0 +1,16 @@ +#ifndef _MLIB_INCLUDE_H_ +#define _MLIB_INCLUDE_H_ + +#ifdef _DEBUG +#define MLIB_ERROR_CHECK +#endif + +#include +#include +#include +#include +#include + +using namespace ml; + +#endif \ No newline at end of file diff --git a/Alignment/src/mLibSource.cpp b/Alignment/src/mLibSource.cpp new file mode 100644 index 0000000..f836f68 --- /dev/null +++ b/Alignment/src/mLibSource.cpp @@ -0,0 +1,7 @@ + +#include "main.h" + +#include "mLibCore.cpp" +#include "mLibLodePNG.cpp" +#include "mLibDepthCamera.cpp" +//#include "mLibZLib.cpp" \ No newline at end of file diff --git a/Alignment/src/main.cpp b/Alignment/src/main.cpp new file mode 100644 index 0000000..fdbe6f8 --- /dev/null +++ b/Alignment/src/main.cpp @@ -0,0 +1,62 @@ + +#include "stdafx.h" + +#include "main.h" +#include "alignment.h" + +void alignScan(const std::string& sceneFolder, bool forceRealign = false) +{ + std::cout << "aligning: " << sceneFolder << std::endl; + Alignment::alignScan(sceneFolder, forceRealign); +} + +void alignScanFromAlnFile(const std::string& sceneFolder) +{ + std::cout << "aligning: " << sceneFolder << std::endl; + Alignment::alignScanFromAlnFile(sceneFolder); +} + + + +void alignDirectory(const std::string& path, bool forceRealign = false) { + Directory dir(path); + std::vector scenes = dir.getDirectories(); + std::sort(scenes.begin(), scenes.end()); + for (size_t i = 0; i < scenes.size(); i++) { + const std::string& scene = scenes[i]; + const std::string sceneFolder = path + scene; + alignScan(sceneFolder); + //break; + //if (i >= 20) break; + } +} + +int main(int argc, char* argv[]) +{ + try { + if (argc == 2) { //converts a specific scan given by the command line argument + std::string stagingFolder(argv[1]); + alignScan(stagingFolder); + } + else { + throw MLIB_EXCEPTION("requires the path as a command line argument"); + } + } + catch (const std::exception& e) + { + MessageBoxA(NULL, e.what(), "Exception caught", MB_ICONERROR); + exit(EXIT_FAILURE); + } + catch (...) + { + MessageBoxA(NULL, "UNKNOWN EXCEPTION", "Exception caught", MB_ICONERROR); + exit(EXIT_FAILURE); + } + + //std::cout << "" << std::endl; + //getchar(); + + return 0; +} + + diff --git a/Alignment/src/main.h b/Alignment/src/main.h new file mode 100644 index 0000000..9e593e5 --- /dev/null +++ b/Alignment/src/main.h @@ -0,0 +1,9 @@ + +#include +#include +#include +#include +#include +#include + +#include "stdafx.h" diff --git a/Alignment/src/planeExtract.h b/Alignment/src/planeExtract.h new file mode 100644 index 0000000..3db2dbb --- /dev/null +++ b/Alignment/src/planeExtract.h @@ -0,0 +1,203 @@ + +#pragma once + +#include "stdafx.h" + + +struct Point { + Planef plane; + vec3f point; + size_t urIdx; +}; + +struct Cluster { + Cluster(const Point* init) { + m_points.push_back(init); + m_rep = *m_points.front(); + + sumNormal = init->plane.getNormal(); + sumPoint = init->point; + } + void addPoint(const Point* p) { + + m_points.push_back(p); + + sumNormal += p->plane.getNormal(); + sumPoint += p->point; + + m_rep.plane = Planef(sumNormal.getNormalized(), sumPoint / (float)m_points.size()); + } + + bool check(const Point& p, float normalThresh, float distThresh) { + float d_norm = p.plane.getNormal() | m_rep.plane.getNormal(); + float d_dist = m_rep.plane.distanceToPointAbs(p.point); + + return (d_norm > normalThresh) && (d_dist < distThresh); + } + + mat4f reComputeNormalAlignment() const { + std::vector points; + for (const Point* p : m_points) { + if (m_rep.plane.distanceToPointAbs(p->point) < 0.05f) { + points.push_back(p->point); + } + } + auto res = math::pointSetPCA(points); + mat4f m(res[0].first.getNormalized(), res[1].first.getNormalized(), res[2].first.getNormalized()); + return m.getTranspose(); + } + + vec3f reComputeNormal() const { + std::vector points; + for (const Point* p : m_points) { + if (m_rep.plane.distanceToPointAbs(p->point) < 0.05f) { + points.push_back(p->point); + } + } + auto res = math::pointSetPCA(points); + mat3f m(res[0].first.getNormalized(), res[1].first.getNormalized(), res[2].first.getNormalized()); + return m.zcol().getNormalized(); + } + + bool operator<(const Cluster& other) const { + return m_points.size() > other.m_points.size(); + } + + Point m_rep; + std::list m_points; + + vec3f sumNormal; + vec3f sumPoint; +}; + +class PlaneExtract { +public: + PlaneExtract(const MeshDataf& md) { + if (!md.hasNormals()) throw MLIB_EXCEPTION("need to compute normals first"); + + m_points.resize(md.m_Vertices.size()); + for (size_t i = 0; i < md.m_Vertices.size(); i++) { + Point& p = m_points[i]; + p.plane = Planef(md.m_Normals[i], md.m_Vertices[i]); + p.point = md.m_Vertices[i]; + p.urIdx = i; + m_pointsRef.push_back(&m_points[i]); + } + } + + void cluster(float normalThresh = 0.90f, float distThresh = 0.05f) { + for (const Point* p : m_pointsRef) { + bool foundCluster = false; + for (Cluster& c : m_clusters) { + if (c.check(*p, normalThresh, distThresh)) { + c.addPoint(p); + foundCluster = true; + break; + } + } + if (!foundCluster) { + m_clusters.push_back(Cluster(p)); + } + } + + m_clusters.sort(); + } + + void print(unsigned int topN = 10) const { + std::cout << m_points.size() << " points " << std::endl; + std::cout << m_clusters.size() << " clusters " << std::endl; + unsigned int i = 0; + for (const Cluster& c : m_clusters) { + if (i >= topN) break; + std::cout << "\t" << c.m_points.size() << std::endl; + i++; + } + } + + + const std::list getClusters() const { + return m_clusters; + } + + void removeSmallClusters(size_t minSize = 500) { + for (std::list::iterator iter = m_clusters.begin(); iter != m_clusters.end();) { + if (iter->m_points.size() < minSize) { + iter = m_clusters.erase(iter); + } + else { + iter++; + } + } + } + + //threshold is 10cm by default + void removeNonBoundingClusters(float distThresh, unsigned int numthresh) { + for (std::list::iterator iter = m_clusters.begin(); iter != m_clusters.end();) { + if (!isBoundingCluster(*iter, distThresh, numthresh)) { + iter = m_clusters.erase(iter); + } + else { + iter++; + } + } + } + + //check whether there are points behind the plane -- if so then return false + bool isBoundingCluster(const Cluster& c, float distThresh, unsigned int numThresh) const { + unsigned int countBehind = 0; + for (size_t i = 0; i < m_points.size(); i++) { + const float d = c.m_rep.plane.distanceToPoint(m_points[i].point); + if (d < -distThresh) countBehind++; + } + if (countBehind > numThresh) return false; + else return true; + } + + void saveColoredPlanes(const std::string& filename, const MeshDataf& md, unsigned int topN = 0) { + topN = (unsigned int)std::min(topN, m_clusters.size()); + + MeshDataf mesh = md; + + if (mesh.m_Colors.size() != mesh.m_Vertices.size()) { + mesh.m_Colors.resize(mesh.m_Vertices.size()); + } + + size_t i = 0; + for (const Cluster& c : m_clusters) { + RGBColor color = RGBColor::randomColor(); + for (const Point* p : c.m_points) { + mesh.m_Colors[p->urIdx] = color; + } + + i++; + if (topN != 0 && i >= topN) { + break; + } + } + + MeshIOf::saveToFile(filename, mesh); + } + + + void saveColoredPlane(const std::string& filename, const Cluster& c, const MeshDataf& md, const RGBColor& color = RGBColor::randomColor()) { + + MeshDataf mesh = md; + + if (mesh.m_Colors.size() != mesh.m_Vertices.size()) { + mesh.m_Colors.resize(mesh.m_Vertices.size()); + } + + for (const Point* p : c.m_points) { + mesh.m_Colors[p->urIdx] = color; + } + + MeshIOf::saveToFile(filename, mesh); + } + +private: + + + std::vector m_points; + std::list m_pointsRef; + std::list m_clusters; +}; diff --git a/Alignment/src/processedFile.h b/Alignment/src/processedFile.h new file mode 100644 index 0000000..3930691 --- /dev/null +++ b/Alignment/src/processedFile.h @@ -0,0 +1,87 @@ +#pragma once + + +#include "mLibInclude.h" + +#define X_PROCESSED_FILE_STATE_FIELDS \ + X(bool, valid) \ + X(unsigned int, heapFreeCount) \ + X(unsigned int, numValidOptTransforms) \ + X(unsigned int, numTransforms) \ + X(bool, aligned) + + +#ifndef VAR_NAME +#define VAR_NAME(x) #x +#endif + +#define checkSizeArray(a, d)( (((sizeof a)/(sizeof a[0])) >= d)) + +class ProcessedFile +{ +public: + +#define X(type, name) type name; + X_PROCESSED_FILE_STATE_FIELDS +#undef X + + //! sets the parameter file and reads + void readMembers(const ParameterFile& parameterFile) { + m_ParameterFile = parameterFile; + readMembers(); + } + + //! reads all the members from the given parameter file (could be called for reloading) + void readMembers() { + aligned = false; + +#define X(type, name) \ + if (!m_ParameterFile.readParameter(std::string(#name), name) && std::string(#name) != "aligned") {MLIB_WARNING(std::string(#name).append(" ").append("uninitialized")); name = type();} + X_PROCESSED_FILE_STATE_FIELDS +#undef X + + m_bIsInitialized = true; + } + + template + std::string makeString(const T& in) { + std::string ret = std::to_string(in); + return ret; + } + template <> + std::string makeString(const bool& in) { + if (in == true) return "true"; + else return "false"; + } + + void saveToFile(const std::string& outFile) { + std::ofstream out(outFile); +#define X(type, name) \ + { out << #name << " = " << makeString(name) << std::endl; } + X_PROCESSED_FILE_STATE_FIELDS +#undef X + } + + void print() const { +#define X(type, name) \ + std::cout << #name " = " << name << std::endl; + X_PROCESSED_FILE_STATE_FIELDS +#undef X + } + + + //! constructor + ProcessedFile() { + m_bIsInitialized = false; + } + + //! destructor + ~ProcessedFile() { + } + + +private: + bool m_bIsInitialized; + ParameterFile m_ParameterFile; +}; + diff --git a/Alignment/src/stdafx.cpp b/Alignment/src/stdafx.cpp new file mode 100644 index 0000000..3f38672 --- /dev/null +++ b/Alignment/src/stdafx.cpp @@ -0,0 +1,11 @@ +// stdafx.cpp : source file that includes just the standard includes +// VirtualScan.pch will be the pre-compiled header +// stdafx.obj will contain the pre-compiled type information + +#pragma warning (disable : 4996) + +#include "stdafx.h" + +#include "mLibSource.cpp" +// TODO: reference any additional headers you need in STDAFX.H +// and not in this file diff --git a/Alignment/src/stdafx.h b/Alignment/src/stdafx.h new file mode 100644 index 0000000..c695e5c --- /dev/null +++ b/Alignment/src/stdafx.h @@ -0,0 +1,13 @@ +// stdafx.h : include file for standard system include files, +// or project specific include files that are used frequently, but +// are changed infrequently +// + +#pragma once + +#include +#include + +// TODO: reference additional headers your program requires here + +#include "mLibInclude.h" \ No newline at end of file diff --git a/BenchmarkScripts/2d_evaluation/evalInstanceLevelSemanticLabeling.py b/BenchmarkScripts/2d_evaluation/evalInstanceLevelSemanticLabeling.py index 1dda4d8..3e3d95e 100644 --- a/BenchmarkScripts/2d_evaluation/evalInstanceLevelSemanticLabeling.py +++ b/BenchmarkScripts/2d_evaluation/evalInstanceLevelSemanticLabeling.py @@ -604,6 +604,7 @@ def evaluateImgLists(predictionList, groundTruthList): def main(argv): #pred_files = [os.path.join(opt.pred_path, file) for file in os.listdir(opt.pred_path) if not os.path.isdir(os.path.join(opt.pred_path, file)) and file.endswith('.txt')] + pred_files = [] # find all the text files recursively for root, dirs, files in os.walk(opt.pred_path): for file in files: @@ -617,9 +618,8 @@ def main(argv): if not os.path.isfile(gt_file): printError("Result file {} does not match any gt file".format(pred_files[i]), user_fault=True) gt_files.append(gt_file) - pred_files[i] = os.path.join(opt.pred_path, pred_files[i]) - print pred_files - print gt_files + #print pred_files + #print gt_files # print some info for user print "Note that this tool uses the file '{}' to cache the ground truth instances.".format(opt.gtInstancesFile) diff --git a/BenchmarkScripts/2d_evaluation/instance.py b/BenchmarkScripts/2d_evaluation/instance.py index 773ff36..562a69f 100644 --- a/BenchmarkScripts/2d_evaluation/instance.py +++ b/BenchmarkScripts/2d_evaluation/instance.py @@ -18,7 +18,7 @@ def __init__(self, imgNp, instID): self.pixelCount = int(self.getInstancePixels(imgNp, instID)) def getLabelID(self, instID): - return int(instID // 256) + return int(instID // 1000) def getInstancePixels(self, imgNp, instLabel): return (imgNp == instLabel).sum() diff --git a/BenchmarkScripts/3d_evaluation/evaluate_semantic_instance.py b/BenchmarkScripts/3d_evaluation/evaluate_semantic_instance.py index 86b24f2..a70d98a 100644 --- a/BenchmarkScripts/3d_evaluation/evaluate_semantic_instance.py +++ b/BenchmarkScripts/3d_evaluation/evaluate_semantic_instance.py @@ -63,7 +63,7 @@ ID_TO_LABEL[VALID_CLASS_IDS[i]] = CLASS_LABELS[i] # ---------- Evaluation params ---------- # # overlaps for evaluation -opt.overlaps = np.arange(0.5,1.,0.05) +opt.overlaps = np.append(np.arange(0.5,0.95,0.05), 0.25) # minimum region size for evaluation [verts] opt.min_region_sizes = np.array( [ 100 ] ) # distance thresholds [m] @@ -82,6 +82,13 @@ def evaluate_matches(matches): ap = np.zeros( (len(dist_threshes) , len(CLASS_LABELS) , len(overlaps)) , np.float ) for di, (min_region_size, distance_thresh, distance_conf) in enumerate(zip(min_region_sizes, dist_threshes, dist_confs)): for oi, overlap_th in enumerate(overlaps): + pred_visited = {} + for m in matches: + for p in matches[m]['pred']: + for label_name in CLASS_LABELS: + for p in matches[m]['pred'][label_name]: + if 'filename' in p: + pred_visited[p['filename']] = False for li, label_name in enumerate(CLASS_LABELS): y_true = np.empty(0) y_score = np.empty(0) @@ -104,7 +111,11 @@ def evaluate_matches(matches): # collect matches for (gti,gt) in enumerate(gt_instances): found_match = False + num_pred = len(gt['matched_pred']) for pred in gt['matched_pred']: + # greedy assignments + if pred_visited[pred['filename']]: + continue overlap = float(pred['intersection']) / (gt['vert_count']+pred['vert_count']-pred['intersection']) if overlap > overlap_th: confidence = pred['confidence'] @@ -123,8 +134,9 @@ def evaluate_matches(matches): found_match = True cur_match[gti] = True cur_score[gti] = confidence + pred_visited[pred['filename']] = True if not found_match: - hardFns += 1 + hard_false_negatives += 1 # remove non-matched ground truth instances cur_true = cur_true [ cur_match==True ] cur_score = cur_score[ cur_match==True ] @@ -135,16 +147,16 @@ def evaluate_matches(matches): for gt in pred['matched_gt']: overlap = float(gt['intersection']) / (gt['vert_count']+pred['vert_count']-gt['intersection']) if overlap > overlap_th: - foundGt = True + found_gt = True break - if not foundGt: + if not found_gt: num_ignore = pred['void_intersection'] for gt in pred['matched_gt']: # group? if gt['instance_id'] < 1000: num_ignore += gt['intersection'] # small ground truth instances - if gt['vert_count'] < min_region_size or gt['med_dist']>distance_thresh or gt['dist_conf']distance_thresh or gt['dist_conf']15.3f}".format(ap_avg ) + sep line += sep + "{:>15.3f}".format(ap_50o ) + sep + line += sep + "{:>15.3f}".format(ap_25o ) + sep print line all_ap_avg = avgs["all_ap"] all_ap_50o = avgs["all_ap_50%"] + all_ap_25o = avgs["all_ap_25%"] print "-"*lineLen line = "{:<15}".format("average") + sep + col1 line += "{:>15.3f}".format(all_ap_avg) + sep line += "{:>15.3f}".format(all_ap_50o) + sep + line += "{:>15.3f}".format(all_ap_25o) + sep print line print "" @@ -325,13 +350,14 @@ def print_results(avgs): def write_result_file(avgs, filename): _SPLITTER = ',' with open(filename, 'w') as f: - f.write(_SPLITTER.join(['class', 'class id', 'ap', 'ap50']) + '\n') + f.write(_SPLITTER.join(['class', 'class id', 'ap', 'ap50', 'ap25']) + '\n') for i in range(len(VALID_CLASS_IDS)): class_name = CLASS_LABELS[i] class_id = VALID_CLASS_IDS[i] ap = avgs["classes"][class_name]["ap"] ap50 = avgs["classes"][class_name]["ap50%"] - f.write(_SPLITTER.join([str(x) for x in [class_name, class_id, ap, ap50]]) + '\n') + ap25 = avgs["classes"][class_name]["ap25%"] + f.write(_SPLITTER.join([str(x) for x in [class_name, class_id, ap, ap50, ap25]]) + '\n') def evaluate(pred_files, gt_files, pred_path, output_file): diff --git a/BenchmarkScripts/3d_evaluation/evaluate_semantic_label.py b/BenchmarkScripts/3d_evaluation/evaluate_semantic_label.py index 883e5aa..0a2f6b6 100644 --- a/BenchmarkScripts/3d_evaluation/evaluate_semantic_label.py +++ b/BenchmarkScripts/3d_evaluation/evaluate_semantic_label.py @@ -31,7 +31,7 @@ import util_3d parser = argparse.ArgumentParser() -parser.add_argument('--pred_path', required=True, help='path to directory of predicted grids and world2grids as np arrays') +parser.add_argument('--pred_path', required=True, help='path to directory of predicted .txt files') parser.add_argument('--gt_path', required=True, help='path to gt files') parser.add_argument('--output_file', default='', help='output file [default: pred_path/semantic_label_evaluation.txt]') opt = parser.parse_args() diff --git a/BenchmarkScripts/ScanNet200/README.md b/BenchmarkScripts/ScanNet200/README.md new file mode 100644 index 0000000..f175272 --- /dev/null +++ b/BenchmarkScripts/ScanNet200/README.md @@ -0,0 +1,60 @@ +# ScanNet200 Preprocessing Scripts and description + +We provide the preprocessing scripts and benchmark data for the ScanNet200 Benchmark. +The raw scans and annotations are shared with the original [ScanNet benchmark](http://kaldir.vc.in.tum.de/scannet_benchmark); these scripts provided output semantic and instance labeled meshes according to the ScanNet200 categories. +The ScanNet scene meshes are surface annotated, where every vertex is described with the raw category id. +These IDs can be parsed based on the mapping defined in the `scannetv2-labels.combined.tsv`. + +**Important Note:** The `scannetv2-labels.combined.tsv` file was updated with the introduction of the ScanNet200 benchmark, please download the latest version using the script obtained after filling the [Terms of Use form](https://github.com/ScanNet/ScanNet#scannet-data). + + +Differences and similarities to the original benchmark + - The ScanNet200 benchmark evaluates 200 categories, an order of magnitude larger than the original set of 20 classical semantic labels. + - This new benchmark follows the original _train_/_val_/_test_ scene splits published in this repository, + - We presented a further split of the category sets into three sets based on their point and instance frequencies, namely **head**, **common**, and **tail**. The category splits can be found in `scannet200_split.py` file + - The raw annotations in the training set containing 550 distinct categories, many of which appear only once, and were filtered to produce the large-vocabulary, challenging ScanNet200 setting. The mapping of annotation category IDs to ScanNet200 valid categories can be found in `scannet200_constants.py`. + - This larger vocabulary includes a strong natural imbalance and diversity for evaluating modern 3D scene understanding methods in a challenging scenario. + +![](docs/dataset_histograms.jpg) + +We provide scripts for preprocessing and parsing the scene meshes to semantically and instance labeled meshes in `preprocess_scannet200.py`. +Additionally, helper functions such as mesh voxelization can be found in `utils.py` + +### Running the preprocessing + +The scripts are developed and tested with Python 3, and basic libraries like _pandas_ and _plyfile_ are expected to be installed. +Additionally, we rely on _trimesh_ and _MinkowskiEngine_ for uniform mesh voxelization, but these libraries are not strictly necessary + +For the installation of all required libraries + +``` +conda create -n scannet200 python=3.8 +pip install -r requirements.txt +``` + +For the optional MinkowskiEngine required in the voxelization script, please refer to the [installation guide](https://github.com/NVIDIA/MinkowskiEngine#anaconda) corresponding the specific GPU version. + + +The preprocessing can be started with + +``` +python --dataset_root + --output_root + --label_map_file +``` + +Where script options: +``` +--dataset_root: + Path to the ScanNet dataset containing scene folders +--output_root: + Output path where train/val folders will be located +--label_map_file: + path to the updated scannetv2-labels.combined.tsv +--num_workers: + The number of parallel workers for multiprocessing + default=4 +--train_val_splits_path: + Where the txt files with the train/val splits live + default='../../Tasks/Benchmark' +``` diff --git a/BenchmarkScripts/ScanNet200/docs/dataset_histograms.jpg b/BenchmarkScripts/ScanNet200/docs/dataset_histograms.jpg new file mode 100644 index 0000000..227baaa Binary files /dev/null and b/BenchmarkScripts/ScanNet200/docs/dataset_histograms.jpg differ diff --git a/BenchmarkScripts/ScanNet200/preprocess_scannet200.py b/BenchmarkScripts/ScanNet200/preprocess_scannet200.py new file mode 100644 index 0000000..6f0e2bb --- /dev/null +++ b/BenchmarkScripts/ScanNet200/preprocess_scannet200.py @@ -0,0 +1,133 @@ +import warnings +warnings.filterwarnings("ignore", category=DeprecationWarning) + +import sys +import os +import argparse +import glob +import json +from concurrent.futures import ProcessPoolExecutor +from itertools import repeat + +# Load external constants +from scannet200_constants import * +from scannet200_splits import * +from utils import * + +CLOUD_FILE_PFIX = '_vh_clean_2' +SEGMENTS_FILE_PFIX = '.0.010000.segs.json' +AGGREGATIONS_FILE_PFIX = '.aggregation.json' +CLASS_IDs = VALID_CLASS_IDS_200 + +def handle_process(scene_path, output_path, labels_pd, train_scenes, val_scenes): + + scene_id = scene_path.split('/')[-1] + mesh_path = os.path.join(scene_path, f'{scene_id}{CLOUD_FILE_PFIX}.ply') + segments_file = os.path.join(scene_path, f'{scene_id}{CLOUD_FILE_PFIX}{SEGMENTS_FILE_PFIX}') + aggregations_file = os.path.join(scene_path, f'{scene_id}{AGGREGATIONS_FILE_PFIX}') + info_file = os.path.join(scene_path, f'{scene_id}.txt') + + if scene_id in train_scenes: + output_file = os.path.join(output_path, 'train', f'{scene_id}.ply') + split_name = 'train' + elif scene_id in val_scenes: + output_file = os.path.join(output_path, 'val', f'{scene_id}.ply') + split_name = 'val' + else: + output_file = os.path.join(output_path, 'test', f'{scene_id}.ply') + split_name = 'test' + + print('Processing: ', scene_id, 'in ', split_name) + + # Rotating the mesh to axis aligned + info_dict = {} + with open(info_file) as f: + for line in f: + (key, val) = line.split(" = ") + info_dict[key] = np.fromstring(val, sep=' ') + + if 'axisAlignment' not in info_dict: + rot_matrix = np.identity(4) + else: + rot_matrix = info_dict['axisAlignment'].reshape(4, 4) + + pointcloud, faces_array = read_plymesh(mesh_path) + points = pointcloud[:, :3] + colors = pointcloud[:, 3:6] + alphas = pointcloud[:, -1] + + # Rotate PC to axis aligned + r_points = pointcloud[:, :3].transpose() + r_points = np.append(r_points, np.ones((1, r_points.shape[1])), axis=0) + r_points = np.dot(rot_matrix, r_points) + pointcloud = np.append(r_points.transpose()[:, :3], pointcloud[:, 3:], axis=1) + + # Load segments file + with open(segments_file) as f: + segments = json.load(f) + seg_indices = np.array(segments['segIndices']) + + # Load Aggregations file + with open(aggregations_file) as f: + aggregation = json.load(f) + seg_groups = np.array(aggregation['segGroups']) + + # Generate new labels + labelled_pc = np.zeros((pointcloud.shape[0], 1)) + instance_ids = np.zeros((pointcloud.shape[0], 1)) + for group in seg_groups: + segment_points, p_inds, label_id = point_indices_from_group(pointcloud, seg_indices, group, labels_pd, CLASS_IDs) + + labelled_pc[p_inds] = label_id + instance_ids[p_inds] = group['id'] + + labelled_pc = labelled_pc.astype(int) + instance_ids = instance_ids.astype(int) + + # Concatenate with original cloud + processed_vertices = np.hstack((pointcloud[:, :6], labelled_pc, instance_ids)) + + if (np.any(np.isnan(processed_vertices)) or not np.all(np.isfinite(processed_vertices))): + raise ValueError('nan') + + # Save processed mesh + save_plymesh(processed_vertices, faces_array, output_file, with_label=True, verbose=False) + + # Uncomment the following lines if saving the output in voxelized point cloud + # quantized_points, quantized_scene_colors, quantized_labels, quantized_instances = voxelize_pointcloud(points, colors, labelled_pc, instance_ids, faces_array) + # quantized_pc = np.hstack((quantized_points, quantized_scene_colors, quantized_labels, quantized_instances)) + # save_plymesh(quantized_pc, faces=None, filename=output_file, with_label=True, verbose=False) + +if __name__ == '__main__': + parser = argparse.ArgumentParser() + parser.add_argument('--dataset_root', required=True, help='Path to the ScanNet dataset containing scene folders') + parser.add_argument('--output_root', required=True, help='Output path where train/val folders will be located') + parser.add_argument('--label_map_file', required=True, help='path to scannetv2-labels.combined.tsv') + parser.add_argument('--num_workers', default=4, type=int, help='The number of parallel workers') + parser.add_argument('--train_val_splits_path', default='../../Tasks/Benchmark', help='Where the txt files with the train/val splits live') + config = parser.parse_args() + + # Load label map + labels_pd = pd.read_csv(config.label_map_file, sep='\t', header=0) + + # Load train/val splits + with open(config.train_val_splits_path + '/scannetv2_train.txt') as train_file: + train_scenes = train_file.read().splitlines() + with open(config.train_val_splits_path + '/scannetv2_val.txt') as val_file: + val_scenes = val_file.read().splitlines() + + # Create output directories + train_output_dir = os.path.join(config.output_root, 'train') + if not os.path.exists(train_output_dir): + os.makedirs(train_output_dir) + val_output_dir = os.path.join(config.output_root, 'val') + if not os.path.exists(val_output_dir): + os.makedirs(val_output_dir) + + # Load scene paths + scene_paths = sorted(glob.glob(config.dataset_root + '/*')) + + # Preprocess data. + pool = ProcessPoolExecutor(max_workers=config.num_workers) + print('Processing scenes...') + _ = list(pool.map(handle_process, scene_paths, repeat(config.output_root), repeat(labels_pd), repeat(train_scenes), repeat(val_scenes))) diff --git a/BenchmarkScripts/ScanNet200/requirements.txt b/BenchmarkScripts/ScanNet200/requirements.txt new file mode 100644 index 0000000..ccf5e0a --- /dev/null +++ b/BenchmarkScripts/ScanNet200/requirements.txt @@ -0,0 +1,15 @@ +certifi==2022.5.18.1 +joblib==1.1.0 +numpy==1.22.4 +pandas==1.4.2 +pip==21.2.4 +plyfile==0.7.4 +python-dateutil==2.8.2 +pytz==2022.1 +scikit-learn==1.1.1 +scipy==1.8.1 +setuptools==61.2.0 +six==1.16.0 +threadpoolctl==3.1.0 +trimesh==3.12.6 +wheel==0.37.1 diff --git a/BenchmarkScripts/ScanNet200/scannet200_constants.py b/BenchmarkScripts/ScanNet200/scannet200_constants.py new file mode 100644 index 0000000..388c497 --- /dev/null +++ b/BenchmarkScripts/ScanNet200/scannet200_constants.py @@ -0,0 +1,277 @@ +### ScanNet Benchmark constants ### +VALID_CLASS_IDS_20 = (1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 14, 16, 24, 28, 33, 34, 36, 39) + +CLASS_LABELS_20 = ('wall', 'floor', 'cabinet', 'bed', 'chair', 'sofa', 'table', 'door', 'window', + 'bookshelf', 'picture', 'counter', 'desk', 'curtain', 'refrigerator', + 'shower curtain', 'toilet', 'sink', 'bathtub', 'otherfurniture') + +SCANNET_COLOR_MAP_20 = { + 0: (0., 0., 0.), + 1: (174., 199., 232.), + 2: (152., 223., 138.), + 3: (31., 119., 180.), + 4: (255., 187., 120.), + 5: (188., 189., 34.), + 6: (140., 86., 75.), + 7: (255., 152., 150.), + 8: (214., 39., 40.), + 9: (197., 176., 213.), + 10: (148., 103., 189.), + 11: (196., 156., 148.), + 12: (23., 190., 207.), + 14: (247., 182., 210.), + 15: (66., 188., 102.), + 16: (219., 219., 141.), + 17: (140., 57., 197.), + 18: (202., 185., 52.), + 19: (51., 176., 203.), + 20: (200., 54., 131.), + 21: (92., 193., 61.), + 22: (78., 71., 183.), + 23: (172., 114., 82.), + 24: (255., 127., 14.), + 25: (91., 163., 138.), + 26: (153., 98., 156.), + 27: (140., 153., 101.), + 28: (158., 218., 229.), + 29: (100., 125., 154.), + 30: (178., 127., 135.), + 32: (146., 111., 194.), + 33: (44., 160., 44.), + 34: (112., 128., 144.), + 35: (96., 207., 209.), + 36: (227., 119., 194.), + 37: (213., 92., 176.), + 38: (94., 106., 211.), + 39: (82., 84., 163.), + 40: (100., 85., 144.), +} + +### ScanNet200 Benchmark constants ### +VALID_CLASS_IDS_200 = ( +1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 13, 14, 15, 16, 17, 18, 19, 21, 22, 23, 24, 26, 27, 28, 29, 31, 32, 33, 34, 35, 36, 38, 39, 40, 41, 42, 44, 45, 46, 47, 48, 49, 50, 51, 52, 54, 55, 56, 57, 58, 59, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, +72, 73, 74, 75, 76, 77, 78, 79, 80, 82, 84, 86, 87, 88, 89, 90, 93, 95, 96, 97, 98, 99, 100, 101, 102, 103, 104, 105, 106, 107, 110, 112, 115, 116, 118, 120, 121, 122, 125, 128, 130, 131, 132, 134, 136, 138, 139, 140, 141, 145, 148, 154, +155, 156, 157, 159, 161, 163, 165, 166, 168, 169, 170, 177, 180, 185, 188, 191, 193, 195, 202, 208, 213, 214, 221, 229, 230, 232, 233, 242, 250, 261, 264, 276, 283, 286, 300, 304, 312, 323, 325, 331, 342, 356, 370, 392, 395, 399, 408, 417, +488, 540, 562, 570, 572, 581, 609, 748, 776, 1156, 1163, 1164, 1165, 1166, 1167, 1168, 1169, 1170, 1171, 1172, 1173, 1174, 1175, 1176, 1178, 1179, 1180, 1181, 1182, 1183, 1184, 1185, 1186, 1187, 1188, 1189, 1190, 1191) + +CLASS_LABELS_200 = ( +'wall', 'chair', 'floor', 'table', 'door', 'couch', 'cabinet', 'shelf', 'desk', 'office chair', 'bed', 'pillow', 'sink', 'picture', 'window', 'toilet', 'bookshelf', 'monitor', 'curtain', 'book', 'armchair', 'coffee table', 'box', +'refrigerator', 'lamp', 'kitchen cabinet', 'towel', 'clothes', 'tv', 'nightstand', 'counter', 'dresser', 'stool', 'cushion', 'plant', 'ceiling', 'bathtub', 'end table', 'dining table', 'keyboard', 'bag', 'backpack', 'toilet paper', +'printer', 'tv stand', 'whiteboard', 'blanket', 'shower curtain', 'trash can', 'closet', 'stairs', 'microwave', 'stove', 'shoe', 'computer tower', 'bottle', 'bin', 'ottoman', 'bench', 'board', 'washing machine', 'mirror', 'copier', +'basket', 'sofa chair', 'file cabinet', 'fan', 'laptop', 'shower', 'paper', 'person', 'paper towel dispenser', 'oven', 'blinds', 'rack', 'plate', 'blackboard', 'piano', 'suitcase', 'rail', 'radiator', 'recycling bin', 'container', +'wardrobe', 'soap dispenser', 'telephone', 'bucket', 'clock', 'stand', 'light', 'laundry basket', 'pipe', 'clothes dryer', 'guitar', 'toilet paper holder', 'seat', 'speaker', 'column', 'bicycle', 'ladder', 'bathroom stall', 'shower wall', +'cup', 'jacket', 'storage bin', 'coffee maker', 'dishwasher', 'paper towel roll', 'machine', 'mat', 'windowsill', 'bar', 'toaster', 'bulletin board', 'ironing board', 'fireplace', 'soap dish', 'kitchen counter', 'doorframe', +'toilet paper dispenser', 'mini fridge', 'fire extinguisher', 'ball', 'hat', 'shower curtain rod', 'water cooler', 'paper cutter', 'tray', 'shower door', 'pillar', 'ledge', 'toaster oven', 'mouse', 'toilet seat cover dispenser', +'furniture', 'cart', 'storage container', 'scale', 'tissue box', 'light switch', 'crate', 'power outlet', 'decoration', 'sign', 'projector', 'closet door', 'vacuum cleaner', 'candle', 'plunger', 'stuffed animal', 'headphones', 'dish rack', +'broom', 'guitar case', 'range hood', 'dustpan', 'hair dryer', 'water bottle', 'handicap bar', 'purse', 'vent', 'shower floor', 'water pitcher', 'mailbox', 'bowl', 'paper bag', 'alarm clock', 'music stand', 'projector screen', 'divider', +'laundry detergent', 'bathroom counter', 'object', 'bathroom vanity', 'closet wall', 'laundry hamper', 'bathroom stall door', 'ceiling light', 'trash bin', 'dumbbell', 'stair rail', 'tube', 'bathroom cabinet', 'cd case', 'closet rod', +'coffee kettle', 'structure', 'shower head', 'keyboard piano', 'case of water bottles', 'coat rack', 'storage organizer', 'folded chair', 'fire alarm', 'power strip', 'calendar', 'poster', 'potted plant', 'luggage', 'mattress') + +SCANNET_COLOR_MAP_200 = { +0: (0., 0., 0.), +1: (174., 199., 232.), +2: (188., 189., 34.), +3: (152., 223., 138.), +4: (255., 152., 150.), +5: (214., 39., 40.), +6: (91., 135., 229.), +7: (31., 119., 180.), +8: (229., 91., 104.), +9: (247., 182., 210.), +10: (91., 229., 110.), +11: (255., 187., 120.), +13: (141., 91., 229.), +14: (112., 128., 144.), +15: (196., 156., 148.), +16: (197., 176., 213.), +17: (44., 160., 44.), +18: (148., 103., 189.), +19: (229., 91., 223.), +21: (219., 219., 141.), +22: (192., 229., 91.), +23: (88., 218., 137.), +24: (58., 98., 137.), +26: (177., 82., 239.), +27: (255., 127., 14.), +28: (237., 204., 37.), +29: (41., 206., 32.), +31: (62., 143., 148.), +32: (34., 14., 130.), +33: (143., 45., 115.), +34: (137., 63., 14.), +35: (23., 190., 207.), +36: (16., 212., 139.), +38: (90., 119., 201.), +39: (125., 30., 141.), +40: (150., 53., 56.), +41: (186., 197., 62.), +42: (227., 119., 194.), +44: (38., 100., 128.), +45: (120., 31., 243.), +46: (154., 59., 103.), +47: (169., 137., 78.), +48: (143., 245., 111.), +49: (37., 230., 205.), +50: (14., 16., 155.), +51: (196., 51., 182.), +52: (237., 80., 38.), +54: (138., 175., 62.), +55: (158., 218., 229.), +56: (38., 96., 167.), +57: (190., 77., 246.), +58: (208., 49., 84.), +59: (208., 193., 72.), +62: (55., 220., 57.), +63: (10., 125., 140.), +64: (76., 38., 202.), +65: (191., 28., 135.), +66: (211., 120., 42.), +67: (118., 174., 76.), +68: (17., 242., 171.), +69: (20., 65., 247.), +70: (208., 61., 222.), +71: (162., 62., 60.), +72: (210., 235., 62.), +73: (45., 152., 72.), +74: (35., 107., 149.), +75: (160., 89., 237.), +76: (227., 56., 125.), +77: (169., 143., 81.), +78: (42., 143., 20.), +79: (25., 160., 151.), +80: (82., 75., 227.), +82: (253., 59., 222.), +84: (240., 130., 89.), +86: (123., 172., 47.), +87: (71., 194., 133.), +88: (24., 94., 205.), +89: (134., 16., 179.), +90: (159., 32., 52.), +93: (213., 208., 88.), +95: (64., 158., 70.), +96: (18., 163., 194.), +97: (65., 29., 153.), +98: (177., 10., 109.), +99: (152., 83., 7.), +100: (83., 175., 30.), +101: (18., 199., 153.), +102: (61., 81., 208.), +103: (213., 85., 216.), +104: (170., 53., 42.), +105: (161., 192., 38.), +106: (23., 241., 91.), +107: (12., 103., 170.), +110: (151., 41., 245.), +112: (133., 51., 80.), +115: (184., 162., 91.), +116: (50., 138., 38.), +118: (31., 237., 236.), +120: (39., 19., 208.), +121: (223., 27., 180.), +122: (254., 141., 85.), +125: (97., 144., 39.), +128: (106., 231., 176.), +130: (12., 61., 162.), +131: (124., 66., 140.), +132: (137., 66., 73.), +134: (250., 253., 26.), +136: (55., 191., 73.), +138: (60., 126., 146.), +139: (153., 108., 234.), +140: (184., 58., 125.), +141: (135., 84., 14.), +145: (139., 248., 91.), +148: (53., 200., 172.), +154: (63., 69., 134.), +155: (190., 75., 186.), +156: (127., 63., 52.), +157: (141., 182., 25.), +159: (56., 144., 89.), +161: (64., 160., 250.), +163: (182., 86., 245.), +165: (139., 18., 53.), +166: (134., 120., 54.), +168: (49., 165., 42.), +169: (51., 128., 133.), +170: (44., 21., 163.), +177: (232., 93., 193.), +180: (176., 102., 54.), +185: (116., 217., 17.), +188: (54., 209., 150.), +191: (60., 99., 204.), +193: (129., 43., 144.), +195: (252., 100., 106.), +202: (187., 196., 73.), +208: (13., 158., 40.), +213: (52., 122., 152.), +214: (128., 76., 202.), +221: (187., 50., 115.), +229: (180., 141., 71.), +230: (77., 208., 35.), +232: (72., 183., 168.), +233: (97., 99., 203.), +242: (172., 22., 158.), +250: (155., 64., 40.), +261: (118., 159., 30.), +264: (69., 252., 148.), +276: (45., 103., 173.), +283: (111., 38., 149.), +286: (184., 9., 49.), +300: (188., 174., 67.), +304: (53., 206., 53.), +312: (97., 235., 252.), +323: (66., 32., 182.), +325: (236., 114., 195.), +331: (241., 154., 83.), +342: (133., 240., 52.), +356: (16., 205., 144.), +370: (75., 101., 198.), +392: (237., 95., 251.), +395: (191., 52., 49.), +399: (227., 254., 54.), +408: (49., 206., 87.), +417: (48., 113., 150.), +488: (125., 73., 182.), +540: (229., 32., 114.), +562: (158., 119., 28.), +570: (60., 205., 27.), +572: (18., 215., 201.), +581: (79., 76., 153.), +609: (134., 13., 116.), +748: (192., 97., 63.), +776: (108., 163., 18.), +1156: (95., 220., 156.), +1163: (98., 141., 208.), +1164: (144., 19., 193.), +1165: (166., 36., 57.), +1166: (212., 202., 34.), +1167: (23., 206., 34.), +1168: (91., 211., 236.), +1169: (79., 55., 137.), +1170: (182., 19., 117.), +1171: (134., 76., 14.), +1172: (87., 185., 28.), +1173: (82., 224., 187.), +1174: (92., 110., 214.), +1175: (168., 80., 171.), +1176: (197., 63., 51.), +1178: (175., 199., 77.), +1179: (62., 180., 98.), +1180: (8., 91., 150.), +1181: (77., 15., 130.), +1182: (154., 65., 96.), +1183: (197., 152., 11.), +1184: (59., 155., 45.), +1185: (12., 147., 145.), +1186: (54., 35., 219.), +1187: (210., 73., 181.), +1188: (221., 124., 77.), +1189: (149., 214., 66.), +1190: (72., 185., 134.), +1191: (42., 94., 198.), +} + +### For instance segmentation the non-object categories ### +VALID_PANOPTIC_IDS = (1, 3) + +CLASS_LABELS_PANOPTIC = ('wall', 'floor') \ No newline at end of file diff --git a/BenchmarkScripts/ScanNet200/scannet200_splits.py b/BenchmarkScripts/ScanNet200/scannet200_splits.py new file mode 100644 index 0000000..cf3d134 --- /dev/null +++ b/BenchmarkScripts/ScanNet200/scannet200_splits.py @@ -0,0 +1,18 @@ +### This file contains the HEAD - COMMON - TAIL split category ids for ScanNet 200 + +HEAD_CATS_SCANNET_200 = ['tv stand', 'curtain', 'blinds', 'shower curtain', 'bookshelf', 'tv', 'kitchen cabinet', 'pillow', 'lamp', 'dresser', 'monitor', 'object', 'ceiling', 'board', 'stove', 'closet wall', 'couch', 'office chair', 'kitchen counter', 'shower', 'closet', 'doorframe', 'sofa chair', 'mailbox', 'nightstand', 'washing machine', 'picture', 'book', 'sink', 'recycling bin', 'table', 'backpack', 'shower wall', 'toilet', 'copier', 'counter', 'stool', 'refrigerator', 'window', 'file cabinet', 'chair', 'wall', 'plant', 'coffee table', 'stairs', 'armchair', 'cabinet', 'bathroom vanity', 'bathroom stall', 'mirror', 'blackboard', 'trash can', 'stair rail', 'box', 'towel', 'door', 'clothes', 'whiteboard', 'bed', 'floor', 'bathtub', 'desk', 'wardrobe', 'clothes dryer', 'radiator', 'shelf'] +COMMON_CATS_SCANNET_200 = ["cushion", "end table", "dining table", "keyboard", "bag", "toilet paper", "printer", "blanket", "microwave", "shoe", "computer tower", "bottle", "bin", "ottoman", "bench", "basket", "fan", "laptop", "person", "paper towel dispenser", "oven", "rack", "piano", "suitcase", "rail", "container", "telephone", "stand", "light", "laundry basket", "pipe", "seat", "column", "bicycle", "ladder", "jacket", "storage bin", "coffee maker", "dishwasher", "machine", "mat", "windowsill", "bulletin board", "fireplace", "mini fridge", "water cooler", "shower door", "pillar", "ledge", "furniture", "cart", "decoration", "closet door", "vacuum cleaner", "dish rack", "range hood", "projector screen", "divider", "bathroom counter", "laundry hamper", "bathroom stall door", "ceiling light", "trash bin", "bathroom cabinet", "structure", "storage organizer", "potted plant", "mattress"] +TAIL_CATS_SCANNET_200 = ["paper", "plate", "soap dispenser", "bucket", "clock", "guitar", "toilet paper holder", "speaker", "cup", "paper towel roll", "bar", "toaster", "ironing board", "soap dish", "toilet paper dispenser", "fire extinguisher", "ball", "hat", "shower curtain rod", "paper cutter", "tray", "toaster oven", "mouse", "toilet seat cover dispenser", "storage container", "scale", "tissue box", "light switch", "crate", "power outlet", "sign", "projector", "candle", "plunger", "stuffed animal", "headphones", "broom", "guitar case", "dustpan", "hair dryer", "water bottle", "handicap bar", "purse", "vent", "shower floor", "water pitcher", "bowl", "paper bag", "alarm clock", "music stand", "laundry detergent", "dumbbell", "tube", "cd case", "closet rod", "coffee kettle", "shower head", "keyboard piano", "case of water bottles", "coat rack", "folded chair", "fire alarm", "power strip", "calendar", "poster", "luggage"] + + +### Given the different size of the official train and val sets, not all ScanNet200 categories are present in the validation set. +### Here we list of categories with labels and IDs present in both train and validation set, and the remaining categories those are present in train, but not in val +### We dont evaluate on unseen validation categories in this benchmark + +VALID_CLASS_IDS_200_VALIDATION = ('wall', 'chair', 'floor', 'table', 'door', 'couch', 'cabinet', 'shelf', 'desk', 'office chair', 'bed', 'pillow', 'sink', 'picture', 'window', 'toilet', 'bookshelf', 'monitor', 'curtain', 'book', 'armchair', 'coffee table', 'box', 'refrigerator', 'lamp', 'kitchen cabinet', 'towel', 'clothes', 'tv', 'nightstand', 'counter', 'dresser', 'stool', 'cushion', 'plant', 'ceiling', 'bathtub', 'end table', 'dining table', 'keyboard', 'bag', 'backpack', 'toilet paper', 'printer', 'tv stand', 'whiteboard', 'blanket', 'shower curtain', 'trash can', 'closet', 'stairs', 'microwave', 'stove', 'shoe', 'computer tower', 'bottle', 'bin', 'ottoman', 'bench', 'board', 'washing machine', 'mirror', 'copier', 'basket', 'sofa chair', 'file cabinet', 'fan', 'laptop', 'shower', 'paper', 'person', 'paper towel dispenser', 'oven', 'blinds', 'rack', 'plate', 'blackboard', 'piano', 'suitcase', 'rail', 'radiator', 'recycling bin', 'container', 'wardrobe', 'soap dispenser', 'telephone', 'bucket', 'clock', 'stand', 'light', 'laundry basket', 'pipe', 'clothes dryer', 'guitar', 'toilet paper holder', 'seat', 'speaker', 'column', 'ladder', 'bathroom stall', 'shower wall', 'cup', 'jacket', 'storage bin', 'coffee maker', 'dishwasher', 'paper towel roll', 'machine', 'mat', 'windowsill', 'bar', 'toaster', 'bulletin board', 'ironing board', 'fireplace', 'soap dish', 'kitchen counter', 'doorframe', 'toilet paper dispenser', 'mini fridge', 'fire extinguisher', 'ball', 'hat', 'shower curtain rod', 'water cooler', 'paper cutter', 'tray', 'shower door', 'pillar', 'ledge', 'toaster oven', 'mouse', 'toilet seat cover dispenser', 'furniture', 'cart', 'scale', 'tissue box', 'light switch', 'crate', 'power outlet', 'decoration', 'sign', 'projector', 'closet door', 'vacuum cleaner', 'plunger', 'stuffed animal', 'headphones', 'dish rack', 'broom', 'range hood', 'dustpan', 'hair dryer', 'water bottle', 'handicap bar', 'vent', 'shower floor', 'water pitcher', 'mailbox', 'bowl', 'paper bag', 'projector screen', 'divider', 'laundry detergent', 'bathroom counter', 'object', 'bathroom vanity', 'closet wall', 'laundry hamper', 'bathroom stall door', 'ceiling light', 'trash bin', 'dumbbell', 'stair rail', 'tube', 'bathroom cabinet', 'closet rod', 'coffee kettle', 'shower head', 'keyboard piano', 'case of water bottles', 'coat rack', 'folded chair', 'fire alarm', 'power strip', 'calendar', 'poster', 'potted plant', 'mattress') + +CLASS_LABELS_200_VALIDATION = (1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 13, 14, 15, 16, 17, 18, 19, 21, 22, 23, 24, 26, 27, 28, 29, 31, 32, 33, 34, 35, 36, 38, 39, 40, 41, 42, 44, 45, 46, 47, 48, 49, 50, 51, 52, 54, 55, 56, 57, 58, 59, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 82, 84, 86, 87, 88, 89, 90, 93, 95, 96, 97, 98, 99, 100, 101, 102, 103, 104, 105, 106, 107, 110, 112, 115, 116, 118, 120, 122, 125, 128, 130, 131, 132, 134, 136, 138, 139, 140, 141, 145, 148, 154, 155, 156, 157, 159, 161, 163, 165, 166, 168, 169, 170, 177, 180, 185, 188, 191, 193, 195, 202, 208, 213, 214, 229, 230, 232, 233, 242, 250, 261, 264, 276, 283, 300, 304, 312, 323, 325, 342, 356, 370, 392, 395, 408, 417, 488, 540, 562, 570, 609, 748, 776, 1156, 1163, 1164, 1165, 1166, 1167, 1168, 1169, 1170, 1171, 1172, 1173, 1175, 1176, 1179, 1180, 1181, 1182, 1184, 1185, 1186, 1187, 1188, 1189, 1191) + +VALID_CLASS_IDS_200_TRAIN_ONLY = ('bicycle', 'storage container', 'candle', 'guitar case', 'purse', 'alarm clock', 'music stand', 'cd case', 'structure', 'storage organizer', 'luggage') + +CLASS_LABELS_200_TRAIN_ONLY = (121, 221, 286, 331, 399, 572, 581, 1174, 1178, 1183, 1190) diff --git a/BenchmarkScripts/ScanNet200/utils.py b/BenchmarkScripts/ScanNet200/utils.py new file mode 100644 index 0000000..0c6b1f8 --- /dev/null +++ b/BenchmarkScripts/ScanNet200/utils.py @@ -0,0 +1,115 @@ +import os +import numpy as np +from plyfile import PlyData, PlyElement +import pandas as pd + +from scannet200_constants import * + +def read_plymesh(filepath): + """Read ply file and return it as numpy array. Returns None if emtpy.""" + with open(filepath, 'rb') as f: + plydata = PlyData.read(f) + if plydata.elements: + vertices = pd.DataFrame(plydata['vertex'].data).values + faces = np.array([f[0] for f in plydata["face"].data]) + return vertices, faces + +def save_plymesh(vertices, faces, filename, verbose=True, with_label=True): + """Save an RGB point cloud as a PLY file. + + Args: + points_3d: Nx6 matrix where points_3d[:, :3] are the XYZ coordinates and points_3d[:, 4:] are + the RGB values. If Nx3 matrix, save all points with [128, 128, 128] (gray) color. + """ + assert vertices.ndim == 2 + if with_label: + if vertices.shape[1] == 7: + python_types = (float, float, float, int, int, int, int) + npy_types = [('x', 'f4'), ('y', 'f4'), ('z', 'f4'), ('red', 'u1'), ('green', 'u1'), + ('blue', 'u1'), ('label', 'u4')] + + if vertices.shape[1] == 8: + python_types = (float, float, float, int, int, int, int, int) + npy_types = [('x', 'f4'), ('y', 'f4'), ('z', 'f4'), ('red', 'u1'), ('green', 'u1'), + ('blue', 'u1'), ('label', 'u4'), ('instance_id', 'u4')] + + else: + if vertices.shape[1] == 3: + gray_concat = np.tile(np.array([128], dtype=np.uint8), (vertices.shape[0], 3)) + vertices = np.hstack((vertices, gray_concat)) + elif vertices.shape[1] == 6: + python_types = (float, float, float, int, int, int) + npy_types = [('x', 'f4'), ('y', 'f4'), ('z', 'f4'), ('red', 'u1'), ('green', 'u1'), + ('blue', 'u1')] + else: + pass + + vertices_list = [] + for row_idx in range(vertices.shape[0]): + cur_point = vertices[row_idx] + vertices_list.append(tuple(dtype(point) for dtype, point in zip(python_types, cur_point))) + vertices_array = np.array(vertices_list, dtype=npy_types) + elements = [PlyElement.describe(vertices_array, 'vertex')] + + if faces is not None: + faces_array = np.empty(len(faces), dtype=[('vertex_indices', 'i4', (3,))]) + faces_array['vertex_indices'] = faces + elements += [PlyElement.describe(faces_array, 'face')] + + # Write + PlyData(elements).write(filename) + + if verbose is True: + print('Saved point cloud to: %s' % filename) + + +# Map the raw category id to the point cloud +def point_indices_from_group(points, seg_indices, group, labels_pd, CLASS_IDs): + group_segments = np.array(group['segments']) + label = group['label'] + + # Map the category name to id + label_ids = labels_pd[labels_pd['raw_category'] == label]['id'] + label_id = int(label_ids.iloc[0]) if len(label_ids) > 0 else 0 + + # Only store for the valid categories + if not label_id in CLASS_IDs: + label_id = 0 + + # get points, where segment indices (points labelled with segment ids) are in the group segment list + point_IDs = np.where(np.isin(seg_indices, group_segments)) + + return points[point_IDs], point_IDs[0], label_id + + +# Uncomment out if mesh voxelization is required +# import trimesh +# from trimesh.voxel import creation +# from sklearn.neighbors import KDTree +# import MinkowskiEngine as ME + + +# VOXELIZE the scene from sampling on the mesh directly instead of vertices +def voxelize_pointcloud(points, colors, labels, instances, faces, voxel_size=0.2): + + # voxelize mesh first and determine closest labels with KDTree search + trimesh_scene_mesh = trimesh.Trimesh(vertices=points, faces=faces) + voxel_grid = creation.voxelize(trimesh_scene_mesh, voxel_size) + voxel_cloud = np.asarray(voxel_grid.points) + orig_tree = KDTree(points, leaf_size=8) + _, voxel_pc_matches = orig_tree.query(voxel_cloud, k=1) + voxel_pc_matches = voxel_pc_matches.flatten() + + # match colors to voxel ids + points = points[voxel_pc_matches] / voxel_size + colors = colors[voxel_pc_matches] + labels = labels[voxel_pc_matches] + instances = instances[voxel_pc_matches] + + # Voxelize scene + quantized_scene, scene_inds = ME.utils.sparse_quantize(points, return_index=True) + quantized_scene_colors = colors[scene_inds] + quantized_labels = labels[scene_inds] + quantized_instances = instances[scene_inds] + + return quantized_scene, quantized_scene_colors, quantized_labels, quantized_instances diff --git a/BenchmarkScripts/convert2panoptic.py b/BenchmarkScripts/convert2panoptic.py new file mode 100644 index 0000000..d1c919f --- /dev/null +++ b/BenchmarkScripts/convert2panoptic.py @@ -0,0 +1,170 @@ +#!/usr/bin/python +# +# Convert to COCO-style panoptic segmentation format (http://cocodataset.org/#format-data). +# + +# python imports +from __future__ import print_function, absolute_import, division, unicode_literals +import os +import glob +import sys +import argparse +import json +import numpy as np + +# Image processing +from PIL import Image + +EVAL_LABELS = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 14, 16, 24, 28, 33, 34, 36, 39] +EVAL_LABEL_NAMES = ["wall", "floor", "cabinet", "bed", "chair", "sofa", "table", "door", "window", "bookshelf", "picture", "counter", "desk", "curtain", "refrigerator", "shower curtain", "toilet", "sink", "bathtub", "otherfurniture"] +EVAL_LABEL_CATS = ["indoor", "indoor", "furniture", "furniture", "furniture", "furniture", "furniture", "furniture", "furniture", "furniture", "furniture", "furniture", "furniture", "furniture", "appliance", "furniture", "furniture", "appliance", "furniture", "furniture"] +EVAL_LABEL_COLORS = [(174, 199, 232), (152, 223, 138), (31, 119, 180), (255, 187, 120), (188, 189, 34), (140, 86, 75), (255, 152, 150), (214, 39, 40), (197, 176, 213), (148, 103, 189), (196, 156, 148), (23, 190, 207), (247, 182, 210), (219, 219, 141), (255, 127, 14), (158, 218, 229), (44, 160, 44), (112, 128, 144), (227, 119, 194), (82, 84, 163)] + +def splitall(path): + allparts = [] + while 1: + parts = os.path.split(path) + if parts[0] == path: # sentinel for absolute paths + allparts.insert(0, parts[0]) + break + elif parts[1] == path: # sentinel for relative paths + allparts.insert(0, parts[1]) + break + else: + path = parts[0] + allparts.insert(0, parts[1]) + return allparts + +# The main method +def convert2panoptic(scannetPath, outputFolder=None): + + if outputFolder is None: + outputFolder = scannetPath + + # find files + search = os.path.join(scannetPath, "*", "instance", "*.png") + files = glob.glob(search) + files.sort() + # quit if we did not find anything + if not files: + print( + "Did not find any files for using matching pattern {}. Please consult the README.".format(search) + ) + sys.exit(-1) + # a bit verbose + print("Converting {} annotation files.".format(len(files))) + + outputBaseFile = "scannet_panoptic" + outFile = os.path.join(outputFolder, "{}.json".format(outputBaseFile)) + print("Json file with the annotations in panoptic format will be saved in {}".format(outFile)) + panopticFolder = os.path.join(outputFolder, outputBaseFile) + if not os.path.isdir(panopticFolder): + print("Creating folder {} for panoptic segmentation PNGs".format(panopticFolder)) + os.mkdir(panopticFolder) + print("Corresponding segmentations in .png format will be saved in {}".format(panopticFolder)) + + categories = [] + for idx in range(len(EVAL_LABELS)): + label = EVAL_LABELS[idx] + name = EVAL_LABEL_NAMES[idx] + cat = EVAL_LABEL_CATS[idx] + color = EVAL_LABEL_COLORS[idx] + isthing = label > 2 + categories.append({'id': int(label), + 'name': name, + 'color': color, + 'supercategory': cat, + 'isthing': isthing}) + + images = [] + annotations = [] + for progress, f in enumerate(files): + + originalFormat = np.array(Image.open(f)) + + parts = splitall(f) + fileName = parts[-1] + sceneName = parts[-3] + outputFileName = "{}__{}".format(sceneName, fileName) + inputFileName = os.path.join(sceneName, "color", fileName) + imageId = os.path.splitext(outputFileName)[0] + # image entry, id for image is its filename without extension + images.append({"id": imageId, + "width": int(originalFormat.shape[1]), + "height": int(originalFormat.shape[0]), + "file_name": inputFileName}) + + pan_format = np.zeros( + (originalFormat.shape[0], originalFormat.shape[1], 3), dtype=np.uint8 + ) + segmentIds = np.unique(originalFormat) + segmInfo = [] + for segmentId in segmentIds: + isCrowd = 0 + if segmentId < 1000: + semanticId = segmentId + else: + semanticId = segmentId // 1000 + if semanticId not in EVAL_LABELS: + continue + + mask = originalFormat == segmentId + color = [segmentId % 256, segmentId // 256, segmentId // 256 // 256] + pan_format[mask] = color + + area = np.sum(mask) # segment area computation + + # bbox computation for a segment + hor = np.sum(mask, axis=0) + hor_idx = np.nonzero(hor)[0] + x = hor_idx[0] + width = hor_idx[-1] - x + 1 + vert = np.sum(mask, axis=1) + vert_idx = np.nonzero(vert)[0] + y = vert_idx[0] + height = vert_idx[-1] - y + 1 + bbox = [int(x), int(y), int(width), int(height)] + + segmInfo.append({"id": int(segmentId), + "category_id": int(semanticId), + "area": int(area), + "bbox": bbox, + "iscrowd": isCrowd}) + + annotations.append({'image_id': imageId, + 'file_name': outputFileName, + "segments_info": segmInfo}) + + Image.fromarray(pan_format).save(os.path.join(panopticFolder, outputFileName)) + + print("\rProgress: {:>3.2f} %".format((progress + 1) * 100 / len(files)), end=' ') + sys.stdout.flush() + + print("\nSaving the json file {}".format(outFile)) + d = {'images': images, + 'annotations': annotations, + 'categories': categories} + with open(outFile, 'w') as f: + json.dump(d, f, sort_keys=True, indent=4) + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument("--dataset-folder", + dest="scannetPath", + help="path to the ScanNet data 'scannet_frames_25k' folder", + required=True, + type=str) + parser.add_argument("--output-folder", + dest="outputFolder", + help="path to the output folder.", + default=None, + type=str) + args = parser.parse_args() + + convert2panoptic(args.scannetPath, args.outputFolder) + + +# call the main +if __name__ == "__main__": + main() diff --git a/BenchmarkScripts/scene_type_evaluation/evaluate_scene_type.py b/BenchmarkScripts/scene_type_evaluation/evaluate_scene_type.py index 1ad5569..922a984 100644 --- a/BenchmarkScripts/scene_type_evaluation/evaluate_scene_type.py +++ b/BenchmarkScripts/scene_type_evaluation/evaluate_scene_type.py @@ -62,22 +62,41 @@ def get_iou(label_id, confusion): return (float(tp) / denom, tp, denom) -def write_result_file(confusion, ious, filename): +def get_acc(label_id, confusion): + if not label_id in VALID_CLASS_IDS: + return float('nan') + # #true positives + tp = np.longlong(confusion[label_id, label_id]) + # #false negatives + fn = np.longlong(confusion[label_id, :].sum()) - tp + denom = (tp + fn) + if denom == 0: + return float('nan') + return (float(tp) / denom, tp, denom) + + +def write_result_file(confusion, ious, accs, filename): with open(filename, 'w') as f: f.write('iou scores\n') for i in range(len(VALID_CLASS_IDS)): label_id = VALID_CLASS_IDS[i] label_name = CLASS_LABELS[i] iou = ious[label_name][0] - f.write('{0:<14s}({1:<2d}): {2:>5.3f}\n'.format(label_name, label_id, iou)) + f.write('{0:<32s}({1:<2d}): {2:>5.3f}\n'.format(label_name, label_id, iou)) + f.write('recall scores\n') + for i in range(len(VALID_CLASS_IDS)): + label_id = VALID_CLASS_IDS[i] + label_name = CLASS_LABELS[i] + acc = accs[label_name][0] + f.write('{0:<32s}({1:<2d}): {2:>5.3f}\n'.format(label_name, label_id, acc)) f.write('\nconfusion matrix\n') f.write('\t\t\t') for i in range(len(VALID_CLASS_IDS)): - #f.write('\t{0:<14s}({1:<2d})'.format(CLASS_LABELS[i], VALID_CLASS_IDS[i])) + #f.write('\t{0:<32s}({1:<2d})'.format(CLASS_LABELS[i], VALID_CLASS_IDS[i])) f.write('{0:<8d}'.format(VALID_CLASS_IDS[i])) f.write('\n') for r in range(len(VALID_CLASS_IDS)): - f.write('{0:<14s}({1:<2d})'.format(CLASS_LABELS[r], VALID_CLASS_IDS[r])) + f.write('{0:<32s}({1:<2d})'.format(CLASS_LABELS[r], VALID_CLASS_IDS[r])) for c in range(len(VALID_CLASS_IDS)): f.write('\t{0:>5.3f}'.format(confusion[VALID_CLASS_IDS[r],VALID_CLASS_IDS[c]])) f.write('\n') @@ -123,18 +142,26 @@ def evaluate(pred_file, gt_file, output_file): confusion[gt_type][pred_type] += 1 class_ious = {} + class_accs = {} for i in range(len(VALID_CLASS_IDS)): label_name = CLASS_LABELS[i] label_id = VALID_CLASS_IDS[i] class_ious[label_name] = get_iou(label_id, confusion) + class_accs[label_name] = get_acc(label_id, confusion) # print print 'classes IoU' print '----------------------------' for i in range(len(VALID_CLASS_IDS)): label_name = CLASS_LABELS[i] - #print('{{0:<14s}: 1:>5.3f}'.format(label_name, class_ious[label_name][0])) - print('{0:<14s}: {1:>5.3f} ({2:>6d}/{3:<6d})'.format(label_name, class_ious[label_name][0], class_ious[label_name][1], class_ious[label_name][2])) - write_result_file(confusion, class_ious, output_file) + #print('{{0:<32s}: 1:>5.3f}'.format(label_name, class_ious[label_name][0])) + print('{0:<32s}: {1:>5.3f} ({2:>6d}/{3:<6d})'.format(label_name, class_ious[label_name][0], class_ious[label_name][1], class_ious[label_name][2])) + print '' + print 'classes recall' + print '----------------------------' + for i in range(len(VALID_CLASS_IDS)): + label_name = CLASS_LABELS[i] + print('{0:<32s}: {1:>5.3f} ({2:>6d}/{3:<6d})'.format(label_name, class_accs[label_name][0], class_accs[label_name][1], class_accs[label_name][2])) + write_result_file(confusion, class_ious, class_accs, output_file) def main(): diff --git a/Calibrate/README.md b/Calibrate/README.md new file mode 100644 index 0000000..303d826 --- /dev/null +++ b/Calibrate/README.md @@ -0,0 +1,17 @@ +## Converts uncalibrated .sens data to color/depth calibrated .sens data +=========================================================================== + +Converts uncalibrated `.sens` file to calibrated `.sens` file; calibrates color and depth distortion and aligns depth with color space. +Uses the calibration output of [CameraParameterEstimation](../CameraParameterEstimation) for a calibrated device. + +### Installation. +This code was developed under VS2013. + +Requirements: +- DirectX SDK June 2010 +- our research library mLib, a git submodule in ../external/mLib +- mLib external libraries can be downloaded [here](https://www.dropbox.com/s/fve3uen5mzonidx/mLibExternal.zip?dl=0) + + +To run: +`calibrate.exe [input sens file] [output sens file] [device calibration map file (from CameraParameterEstimation)] [directory of device calibration map files]` \ No newline at end of file diff --git a/Calibrate/calibrate.sln b/Calibrate/calibrate.sln new file mode 100644 index 0000000..9ac4411 --- /dev/null +++ b/Calibrate/calibrate.sln @@ -0,0 +1,22 @@ + +Microsoft Visual Studio Solution File, Format Version 12.00 +# Visual Studio 2013 +VisualStudioVersion = 12.0.31101.0 +MinimumVisualStudioVersion = 10.0.40219.1 +Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "calibrate", "calibrate.vcxproj", "{EB0D04B5-579C-4BF9-8FC3-58DFB235231F}" +EndProject +Global + GlobalSection(SolutionConfigurationPlatforms) = preSolution + Debug|x64 = Debug|x64 + Release|x64 = Release|x64 + EndGlobalSection + GlobalSection(ProjectConfigurationPlatforms) = postSolution + {EB0D04B5-579C-4BF9-8FC3-58DFB235231F}.Debug|x64.ActiveCfg = Debug|x64 + {EB0D04B5-579C-4BF9-8FC3-58DFB235231F}.Debug|x64.Build.0 = Debug|x64 + {EB0D04B5-579C-4BF9-8FC3-58DFB235231F}.Release|x64.ActiveCfg = Release|x64 + {EB0D04B5-579C-4BF9-8FC3-58DFB235231F}.Release|x64.Build.0 = Release|x64 + EndGlobalSection + GlobalSection(SolutionProperties) = preSolution + HideSolutionNode = FALSE + EndGlobalSection +EndGlobal diff --git a/Calibrate/calibrate.vcxproj b/Calibrate/calibrate.vcxproj new file mode 100644 index 0000000..ba7c9f7 --- /dev/null +++ b/Calibrate/calibrate.vcxproj @@ -0,0 +1,113 @@ + + + + + Debug + x64 + + + Release + x64 + + + + + + + + + + + + + + + + true + true + + + Create + Create + + + + + true + true + + + + {EB0D04B5-579C-4BF9-8FC3-58DFB235231F} + alignment + calibrate + + + + Application + true + v120 + MultiByte + + + Application + false + v120 + true + MultiByte + + + + + + + + + + + + + ../external/mLib/include;../external/mLibExternal/include/;./src/;C:\Program Files (x86)\Microsoft DirectX SDK (June 2010)\Include;$(IncludePath) + ..\external\mLibExternal\libsWindows\lib64;C:\Program Files %28x86%29\Microsoft DirectX SDK %28June 2010%29\Lib\x64;$(LibraryPath) + + + ../external/mLib/include;../external/mLibExternal/include/;./src/;C:\Program Files (x86)\Microsoft DirectX SDK (June 2010)\Include;$(IncludePath) + ..\external\mLibExternal\libsWindows\lib64;C:\Program Files %28x86%29\Microsoft DirectX SDK %28June 2010%29\Lib\x64;$(LibraryPath) + + + + Level3 + Disabled + true + NOMINMAX;_CRT_SECURE_NO_WARNINGS;_DEBUG;%(PreprocessorDefinitions) + Use + -Zm400 %(AdditionalOptions) + + + true + D3DCompiler.lib;FW1FontWrapper.lib;d3d11.lib;d3dx11.lib;kernel32.lib;user32.lib;gdi32.lib;zlib64.lib;freeimage.lib;%(AdditionalDependencies) + + + + + Level2 + MaxSpeed + true + true + true + NOMINMAX;_CRT_SECURE_NO_WARNINGS;%(PreprocessorDefinitions) + true + Use + -Zm400 %(AdditionalOptions) + + + true + true + true + D3DCompiler.lib;FW1FontWrapper.lib;d3d11.lib;d3dx11.lib;kernel32.lib;user32.lib;gdi32.lib;zlib64.lib;freeimage.lib;%(AdditionalDependencies) + + + + + + \ No newline at end of file diff --git a/Calibrate/calibrate.vcxproj.filters b/Calibrate/calibrate.vcxproj.filters new file mode 100644 index 0000000..3c5acfa --- /dev/null +++ b/Calibrate/calibrate.vcxproj.filters @@ -0,0 +1,21 @@ + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/Calibrate/shaders/aligner.hlsl b/Calibrate/shaders/aligner.hlsl new file mode 100644 index 0000000..f8760a1 --- /dev/null +++ b/Calibrate/shaders/aligner.hlsl @@ -0,0 +1,180 @@ + +#define MINF asfloat(0xff800000) + +#define DEPTH_WORLD_MIN 0.1f +#define DEPTH_WORLD_MAX 10.0f + +float cameraToKinectProjZ(float z) +{ + return (z - DEPTH_WORLD_MIN) / (DEPTH_WORLD_MAX - DEPTH_WORLD_MIN); +} + +float kinectProjZToCamera(float z) +{ + return DEPTH_WORLD_MIN + z*(DEPTH_WORLD_MAX - DEPTH_WORLD_MIN); +} + + +Buffer g_depth : register(t0); + +cbuffer ConstantBuffer : register(b0) +{ + matrix g_mIntrinsicInverse; + matrix g_mIntrinsicNew; + matrix g_mExtrinsic; + uint g_uWidth; + uint g_uHeight; + uint g_uWidthNew; + uint g_uHeightNew; + float g_fDepthThreshOffset; + float g_fDepthThreshLin; + float2 g_dummy123; +} + + +void vertexShaderMain() +{ +} + +struct GS_INPUT +{ +}; + +struct PS_INPUT +{ + float4 vPosition : SV_POSITION; + float fDepth : DE; +}; + +struct PS_OUTPUT +{ + float depth; +}; + +float4 getWorldSpacePosition(uint x, uint y) +{ + float d = g_depth[y*g_uWidth + x]; + float4 posCam = mul(float4((float)x*d, (float)y*d, d, d), g_mIntrinsicInverse); + posCam = float4(posCam.x, posCam.y, posCam.z, 1.0f); + + float4 posWorld = mul(posCam, g_mExtrinsic); + posWorld /= posWorld.w; + + return posWorld; +} + +PS_INPUT ComputeQuadVertex(uint x, uint y) +{ + float d = g_depth[y*g_uWidth + x]; + float4 posWorldCC = getWorldSpacePosition(x, y); + + //float4 posWorlMC = getWorldSpacePosition(x - 1, y + 0); + //float4 posWorlCM = getWorldSpacePosition(x + 0, y - 1); + //float4 posWorlCP = getWorldSpacePosition(x + 0, y + 1); + //float4 posWorlPC = getWorldSpacePosition(x + 1, y + 0); + + //float3 normal = cross(posWorlCP.xyz - posWorlCM.xyz, posWorlPC.xyz - posWorlMC.xyz); + // normal = normalize(normal); + + float4 posClip = mul(float4(posWorldCC.x, posWorldCC.y, posWorldCC.z, 1.0f), g_mIntrinsicNew); + posClip = float4(posClip.x / posClip.z, posClip.y / posClip.z, posClip.z, 1.0f); + + // NOTE: The aspect ratio is not 2.0! + float fx = ((float)posClip.x / (float)(g_uWidthNew - 1))*2.0f - 1.0f; + //float fy = ((float)posClip.y / (float)(g_uHeightNew-1))*2.0f - 1.0f; + float fy = 1.0f - ((float) posClip.y / (float) (g_uHeightNew - 1.0f))*2.0f; + float fz = cameraToKinectProjZ(posClip.z); + posClip.x = fx; + posClip.y = fy; + posClip.z = fz; + posClip.w = 1.0f; + + PS_INPUT Out; + //Out.camSpacePosition = posWorldCC; + Out.vPosition = posClip; + //Out.vNormal = float4(normal, 1.0f); + //Out.vTexCoord = float2(0.0f, 0.0f); + Out.fDepth = d; + + return Out; +} + +PS_INPUT ComputeDebugQuadVertex(uint x, uint y) { + float d = g_depth[y*g_uWidth + x]; + + float4 posClip; + float fx = ((float)x / (float)(g_uWidth - 1))*2.0f - 1.0f; + float fy = 1.0f - ((float)y / (float)(g_uHeight - 1))*2.0f; + float fz = cameraToKinectProjZ(d); + + posClip.x = fx; + posClip.y = fy; + posClip.z = fz; + posClip.w = 1.0f; + + PS_INPUT Out; + Out.vPosition = posClip; + Out.fDepth = d; + return Out; +} + +bool isValidVertex(PS_INPUT v) { + if (v.vPosition.x < -1.0f || v.vPosition.x > 1.0f) return false; + if (v.vPosition.y < -1.0f || v.vPosition.y > 1.0f) return false; + if (v.vPosition.z < 0.0f || v.vPosition.z > 1.0f) return false; + + return true; +} + +[maxvertexcount(4)] +void geometryShaderMain(point GS_INPUT fake[1], uint quadIdx : SV_PrimitiveID, inout TriangleStream OutStream) +{ + PS_INPUT Out = (PS_INPUT)0; + + uint x = quadIdx % g_uWidth; + uint y = quadIdx / g_uWidth; + + if (x >= g_uWidth - 1) return; + if (y >= g_uHeight - 1) return; + + float d0 = g_depth[(x + 0) + g_uWidth*(y + 0)]; + float d1 = g_depth[(x + 0) + g_uWidth*(y + 1)]; + float d2 = g_depth[(x + 1) + g_uWidth*(y + 0)]; + float d3 = g_depth[(x + 1) + g_uWidth*(y + 1)]; + + if (d0 <= DEPTH_WORLD_MIN || d1 <= DEPTH_WORLD_MIN || d2 <= DEPTH_WORLD_MIN || d3 <= DEPTH_WORLD_MIN) return; + if (d0 == MINF || d1 == MINF || d2 == MINF || d3 == MINF) return; + + float dmax = max(max(d0, d1), max(d2, d3)); + float dmin = min(min(d0, d1), min(d2, d3)); + + float d = 0.5f*(dmax + dmin); + if (dmax - dmin > g_fDepthThreshOffset + g_fDepthThreshLin*d) return; + + //be aware of the order for CULLING + PS_INPUT v0 = ComputeQuadVertex(x + 0, y + 1); + PS_INPUT v1 = ComputeQuadVertex(x + 0, y + 0); + PS_INPUT v2 = ComputeQuadVertex(x + 1, y + 1); + PS_INPUT v3 = ComputeQuadVertex(x + 1, y + 0); + + if (isValidVertex(v0) && isValidVertex(v1) && isValidVertex(v2) && isValidVertex(v3)) { + OutStream.Append(v0); + OutStream.Append(v1); + OutStream.Append(v2); + OutStream.Append(v3); + } + + //OutStream.Append(ComputeQuadVertex(x + 0, y + 1)); + //OutStream.Append(ComputeQuadVertex(x + 0, y + 0)); + //OutStream.Append(ComputeQuadVertex(x + 1, y + 1)); + //OutStream.Append(ComputeQuadVertex(x + 1, y + 0)); +} + +float4 pixelShaderMain(PS_INPUT input) : SV_Target +{ + PS_OUTPUT res; + res.depth = input.fDepth; + + return float4(res, res, res, 1.0f); +} + diff --git a/Calibrate/src/aligner.cpp b/Calibrate/src/aligner.cpp new file mode 100644 index 0000000..55d6a1c --- /dev/null +++ b/Calibrate/src/aligner.cpp @@ -0,0 +1,4 @@ + +#include "stdafx.h" + +#include "aligner.h" \ No newline at end of file diff --git a/Calibrate/src/aligner.h b/Calibrate/src/aligner.h new file mode 100644 index 0000000..5c8ca1c --- /dev/null +++ b/Calibrate/src/aligner.h @@ -0,0 +1,145 @@ + +#pragma once + +#include "mLibInclude.h" + +class Aligner { +public: + Aligner(GraphicsDevice& g) { + m_graphics = &g; + + m_shaderManager.init(*m_graphics); + std::string shaderPath = util::getExecutablePath() + "../../"; + const std::string shaderFile = "shaders/aligner.hlsl"; + if (!util::fileExists(shaderPath + shaderFile)) shaderPath = util::getExecutablePath(); + m_shaderManager.registerShaderWithGS(shaderPath + shaderFile, "aligner"); + + m_constantBuffer.init(*m_graphics); + } + + ~Aligner() { + + } + + DepthImage32 depthToColor(const DepthImage32& input, const mat4f& depthIntrinsic, const mat4f& depthExtrinsic, const mat4f& colorIntrinsic, unsigned int colorWidth, unsigned int colorHeight) { + + CB cb; + cb.intrinsicInverse = depthIntrinsic.getInverse(); + cb.extrinsic = depthExtrinsic; + cb.intrinsicNew = colorIntrinsic; + cb.width = input.getWidth(); + cb.height = input.getHeight(); + cb.widthNew = colorWidth; + cb.heightNew = colorHeight; + cb.depthThreshLin = 0.05f; // additional discontinuity threshold per meter + cb.depthThreshOffset = 0.01f; // discontinuity offset in meter + m_constantBuffer.update(cb); + + D3D11Buffer depthGPU; + std::vector inputVec(input.getWidth()*input.getHeight()); + for (size_t i = 0; i < inputVec.size(); i++) inputVec[i] = input.getData()[i]; + depthGPU.init(*m_graphics, inputVec); + + D3D11RenderTarget renderTarget; + renderTarget.init(m_graphics->castD3D11(), input.getWidth(), input.getHeight(), std::vector < DXGI_FORMAT > {DXGI_FORMAT_R8G8B8A8_UNORM}, true); + renderTarget.clear(); + renderTarget.bind(); + + ID3D11DeviceContext& context = m_graphics->castD3D11().getContext(); + + context.IASetPrimitiveTopology(D3D11_PRIMITIVE_TOPOLOGY_POINTLIST); + context.IASetInputLayout(NULL); + unsigned int stride = 0; + unsigned int offset = 0; + context.IASetVertexBuffers(0, 0, NULL, &stride, &offset); + + m_shaderManager.bindShaders("aligner"); + depthGPU.bindSRV(0); + m_constantBuffer.bind(0); + + m_graphics->castD3D11().setCullMode(D3D11_CULL_NONE); + unsigned int numQuads = input.getWidth()*input.getHeight(); + context.Draw(numQuads, 0); + + + //! reset the state + m_shaderManager.unbindShaders(); + depthGPU.unbindSRV(0); + + renderTarget.unbind(); + + DepthImage32 res; + res.setInvalidValue(input.getInvalidValue()); + renderTarget.captureDepthBuffer(res); + for (auto& r : res) { + if (r.value == 1.0f) r.value = res.getInvalidValue(); + else r.value = kinectProjZToCamera(r.value); + } + + context.Flush(); + + return res; + } + + //projects a depth image into color space (debug version with aliasing) + static DepthImage32 depthToColorDebug(const DepthImage32& input, const mat4f& depthIntrinsic, const mat4f& depthExtrinsic, const mat4f& colorIntrinsic, unsigned int colorWidth, unsigned int colorHeight) { + + mat4f depthIntrinsicInv = depthIntrinsic.getInverse(); + + float scalarWidth = (float)(input.getWidth()-1) / (float)(colorWidth-1); + float scalarHeight = (float)(input.getHeight()-1) / (float)(colorHeight-1); + + DepthImage32 res(input.getWidth(), input.getHeight()); + res.setInvalidValue(input.getInvalidValue()); + res.setPixels(input.getInvalidValue()); + + for (auto& o : input) { + float d = o.value; + if (d != res.getInvalidValue()) { + vec3f p = depthIntrinsicInv*vec3f((float)o.x*d, (float)o.y*d, d); //back-project to camera space + p = depthExtrinsic * p; //project to color frame + vec3f colorCoord = colorIntrinsic * p; //project to color image space + colorCoord.x /= colorCoord.z; colorCoord.y /= colorCoord.z; + vec2i colorCoordi = math::round(colorCoord); //use that to get color values + colorCoordi.x = math::round(colorCoord.x * scalarWidth); + colorCoordi.y = math::round(colorCoord.y * scalarHeight); + + if (colorCoordi.x >= 0 && colorCoordi.x < (int)res.getWidth() && colorCoordi.y >= 0 && colorCoordi.y < (int)res.getHeight()) { + res(colorCoordi.x, colorCoordi.y) = colorCoord.z; + } + } + } + return res; + } +private: +#define DEPTH_WORLD_MIN 0.1f +#define DEPTH_WORLD_MAX 10.0f + + float cameraToKinectProjZ(float z) + { + return (z - DEPTH_WORLD_MIN) / (DEPTH_WORLD_MAX - DEPTH_WORLD_MIN); + } + + float kinectProjZToCamera(float z) + { + return DEPTH_WORLD_MIN + z*(DEPTH_WORLD_MAX - DEPTH_WORLD_MIN); + } + + GraphicsDevice* m_graphics; + D3D11ShaderManager m_shaderManager; + + struct CB { + mat4f intrinsicInverse; + mat4f intrinsicNew; + mat4f extrinsic; + unsigned int width; + unsigned int height; + unsigned int widthNew; + unsigned int heightNew; + float depthThreshOffset; + float depthThreshLin; + vec2f dummy; + }; + D3D11ConstantBuffer m_constantBuffer; + +}; \ No newline at end of file diff --git a/Calibrate/src/calibration.h b/Calibrate/src/calibration.h new file mode 100644 index 0000000..8f14d93 --- /dev/null +++ b/Calibrate/src/calibration.h @@ -0,0 +1,311 @@ +#pragma once + +#include "stdafx.h" +#include "grid3d.h" + +#include "aligner.h" + +#include "omp.h" + +class Calib { +public: + Calib(const std::string& str) { + reset(); + readFromFile(str); + } + ~Calib() { + } + + void readFromFile(const std::string& str) { + ParameterFile parameterFile(str); + + reset(); + + parameterFile.readParameter("colorWidth", color_width); + parameterFile.readParameter("colorHeight", color_height); + parameterFile.readParameter("fx_color", color_intrinsic(0, 0)); + parameterFile.readParameter("fy_color", color_intrinsic(1, 1)); + parameterFile.readParameter("mx_color", color_intrinsic(0, 2)); + parameterFile.readParameter("my_color", color_intrinsic(1, 2)); + parameterFile.readParameter("k1_color", color_dist_coeff[0]); + parameterFile.readParameter("k2_color", color_dist_coeff[1]); + parameterFile.readParameter("k3_color", color_dist_coeff[2]); + parameterFile.readParameter("k4_color", color_dist_coeff[3]); + parameterFile.readParameter("k5_color", color_dist_coeff[4]); + + parameterFile.readParameter("depthWidth", depth_width); + parameterFile.readParameter("depthHeight", depth_height); + parameterFile.readParameter("fx_depth", depth_intrinsic(0, 0)); + parameterFile.readParameter("fy_depth", depth_intrinsic(1, 1)); + parameterFile.readParameter("mx_depth", depth_intrinsic(0, 2)); + parameterFile.readParameter("my_depth", depth_intrinsic(1, 2)); + parameterFile.readParameter("k1_depth", depth_dist_coeff[0]); + parameterFile.readParameter("k2_depth", depth_dist_coeff[1]); + parameterFile.readParameter("k3_depth", depth_dist_coeff[2]); + parameterFile.readParameter("k4_depth", depth_dist_coeff[3]); + parameterFile.readParameter("k5_depth", depth_dist_coeff[4]); + + parameterFile.readParameter("depthToColorExtrinsics", depth_extrinsic); + } + + unsigned int color_width, color_height; + mat4f color_extrinsic; + mat4f color_intrinsic; + float color_dist_coeff[5]; + + unsigned int depth_width, depth_height; + mat4f depth_intrinsic; + mat4f depth_extrinsic; //depth-to-color map; + float depth_dist_coeff[5]; + +private: + void reset() + { + color_extrinsic.setIdentity(); + color_intrinsic.setIdentity(); + memset(color_dist_coeff, 0, sizeof(float) * 5); + depth_extrinsic.setIdentity(); + depth_intrinsic.setIdentity(); + memset(depth_dist_coeff, 0, sizeof(float) * 5); + } + +}; + +// "Public" interface +class Calibration +{ +public: + Calibration() { + m_graphics = new D3D11GraphicsDevice(); + m_graphics->initWithoutWindow(); + } + ~Calibration() { + SAFE_DELETE(m_graphics); + } + + + void Calibration::calibrateScan(const std::string& inSensFilename, const std::string &outSensFilename, const std::string& parametersFilename, const std::string& undistortTableFilename) + { + if (!util::fileExists(inSensFilename)) { + if (util::fileExists(outSensFilename)) { + std::cout << "output sens file " << outSensFilename << " already exists, skipping" << std::endl; + return; + } + else { + throw MLIB_EXCEPTION("no sens file: " + inSensFilename); + } + } + if (!util::fileExists(parametersFilename) || !util::fileExists(undistortTableFilename)) throw MLIB_EXCEPTION("no calibration param file(s): " + parametersFilename + " / " + undistortTableFilename); + + // Read in lookup table + Grid3D undistortTable(undistortTableFilename); + + + // Read in parameters + Calib cd(parametersFilename); + + // Read in sens file + std::cout << "loading .sens file " << inSensFilename << "... "; + SensorData sd(inSensFilename); + std::cout << "done!" << std::endl; + + if (sd.m_calibrationDepth.m_extrinsic == mat4f::identity()) { + if (inSensFilename != outSensFilename) util::moveFile(inSensFilename, outSensFilename); + std::cout << "color and depth is already aligned -- cannot further calibrate .sens file -> exiting" << std::endl; + return; + } + + calibrateScan(sd, cd, undistortTable); + + // Updating .sens file meta data + sd.m_sensorName = sd.m_sensorName + " (calibrated)"; + sd.m_calibrationColor.m_extrinsic.setIdentity(); + sd.m_calibrationColor.m_intrinsic = cd.color_intrinsic; + sd.m_calibrationDepth.m_extrinsic.setIdentity(); + sd.m_calibrationDepth.m_intrinsic = sd.m_calibrationColor.m_intrinsic; + + sd.m_calibrationDepth.m_intrinsic(0, 0) *= (float)sd.m_depthWidth / (float)sd.m_colorWidth; + sd.m_calibrationDepth.m_intrinsic(1, 1) *= (float)sd.m_depthHeight / (float)sd.m_colorHeight; + sd.m_calibrationDepth.m_intrinsic(0, 2) *= (float)(sd.m_depthWidth - 1) / (float)(sd.m_colorWidth - 1); + sd.m_calibrationDepth.m_intrinsic(1, 2) *= (float)(sd.m_depthHeight - 1) / (float)(sd.m_colorHeight - 1); + + + // write sens file + std::cout << "saving .sens file " << outSensFilename << "... "; + sd.saveToFile(outSensFilename); + if (inSensFilename != outSensFilename) util::deleteFile(inSensFilename); + std::cout << "done!" << std::endl; + } + +private: + + DepthImage32 depthToColor(const DepthImage32& input, const Calib& cb) { + DepthImage32 res; +#pragma omp critical + { + Aligner aligner(*m_graphics); + res = aligner.depthToColor(input, cb.depth_intrinsic, cb.depth_extrinsic, cb.color_intrinsic, cb.color_width, cb.color_height); + } + return res; + } + + //projects a depth image into color space + DepthImage32 depthToColorDebug(const DepthImage32& input, const Calib& cb) { + + mat4f depthIntrinsicInv = cb.depth_intrinsic.getInverse(); + const mat4f& extrinsic = cb.depth_extrinsic; + const mat4f& colorIntrinsic = cb.color_intrinsic; + + float scalarWidth = (float)(input.getWidth()-1) / (float)(cb.color_width-1); + float scalarHeight = (float)(input.getHeight()-1) / (float)(cb.color_height-1); + + DepthImage32 res(input.getWidth(), input.getHeight()); + res.setInvalidValue(input.getInvalidValue()); + res.setPixels(input.getInvalidValue()); + + for (auto& o : input) { + float d = o.value; + if (d != res.getInvalidValue()) { + vec3f p = depthIntrinsicInv*vec3f((float)o.x*d, (float)o.y*d, d); //back-project to camera space + p = extrinsic * p; //project to color frame + vec3f colorCoord = colorIntrinsic * p; //project to color image space + colorCoord.x /= colorCoord.z; colorCoord.y /= colorCoord.z; + vec2i colorCoordi = math::round(colorCoord); //use that to get color values + colorCoordi.x = math::round(colorCoord.x * scalarWidth); + colorCoordi.y = math::round(colorCoord.y * scalarHeight); + + if (colorCoordi.x >= 0 && colorCoordi.x < (int)res.getWidth() && colorCoordi.y >= 0 && colorCoordi.y < (int)res.getHeight()) { + res(colorCoordi.x, colorCoordi.y) = colorCoord.z; + } + } + } + return res; + } + + template BaseImage undistort(const BaseImage& src, const mat4f& intrinsic, const float coeff[5]) + { + BaseImage res(src.getWidth(), src.getHeight()); + res.setInvalidValue(src.getInvalidValue()); + res.setPixels(res.getInvalidValue()); + + for (unsigned int y = 0; y < src.getHeight(); y++) { + for (unsigned int x = 0; x < src.getWidth(); x++) { + vec2f nic_loc; + vec2f sample_loc; + + //Normalized image coords + nic_loc.x = (x - intrinsic(0, 2)) / intrinsic(0, 0); + nic_loc.y = (y - intrinsic(1, 2)) / intrinsic(1, 1); + + float r2 = nic_loc.x * nic_loc.x + nic_loc.y * nic_loc.y; + + // Radial distortion + sample_loc.x = nic_loc.x * (1.0f + r2 * coeff[0] + r2*r2 * coeff[1] + r2*r2*r2 * coeff[4]); + sample_loc.y = nic_loc.y * (1.0f + r2 * coeff[0] + r2*r2 * coeff[1] + r2*r2*r2 * coeff[4]); + + // Tangential distortion + sample_loc.x += 2.0f * coeff[2] * nic_loc.x * nic_loc.y + coeff[3] * (r2 + 2.0f * nic_loc.x * nic_loc.x); + sample_loc.y += coeff[2] * (r2 + 2.0f * nic_loc.y * nic_loc.y) + 2.0f * coeff[3] * nic_loc.x * nic_loc.y; + + // Move back to the image space + sample_loc.x = sample_loc.x * intrinsic(0, 0) + intrinsic(0, 2); + sample_loc.y = sample_loc.y * intrinsic(1, 1) + intrinsic(1, 2); + + //TODO CONTINUE HERE + vec2i sample_loc_i = math::round(sample_loc); + if (src.isValidCoordinate(sample_loc_i)) { + res(x, y) = src(sample_loc_i); + } + } + } + return res; + } + + + void undistortDistance(unsigned short * depthData, const SensorData &sd, const Grid3D& undistortTable) { + + // Prepare storage + // Useful vars + int w = sd.m_depthWidth; + int h = sd.m_depthHeight; + int nSlices = undistortTable.ZRes(); + float xBin = (float)(w / undistortTable.XRes()); + float yBin = (float)(h / undistortTable.YRes()); + float zBin = undistortTable.ZRes() / undistortTable.MaxDist(); + float depthShift = sd.m_depthShift; + + for (int j = 0; j < h; ++j) + { + for (int i = 0; i < w; ++i) + { + float depth = depthData[j * w + i] / depthShift; + float zIdx = std::min((float)depth * zBin, (float)nSlices - 1.0f); + + float multiplier = 1.0f / undistortTable.GetValue((float)i / xBin, (float)j / yBin, zIdx); + float newDepth = depth * multiplier; + depthData[j*w + i] = (unsigned short)(newDepth * depthShift); + + } + } + } + + + void calibrateScan(SensorData& sd, const Calib& cd, const Grid3D& undistortTable) { + if (cd.depth_width != sd.m_depthWidth || cd.depth_height != sd.m_depthHeight) throw MLIB_EXCEPTION("image dimensions do not match with calibration"); + +#pragma omp parallel for + for (int i = 0; i < (int)sd.m_frames.size(); i ++) { + auto& f = sd.m_frames[i]; + if (omp_get_thread_num() == 0) { + std::cout << "\rcalibrateScan frame [ " << i*omp_get_num_threads() << " | " << sd.m_frames.size() << " ] "; + } + + // apply un-distortion to color + vec3uc* color = sd.decompressColorAlloc(f); + ColorImageR8G8B8 c(sd.m_colorWidth, sd.m_colorHeight, color); + c.setInvalidValue(vec3uc(0, 0, 0)); + c = undistort(c, cd.color_intrinsic, cd.color_dist_coeff); + std::free(color); + sd.replaceColor(f, c.getData()); + + + unsigned short* depth = sd.decompressDepthAlloc(f); + undistortDistance(depth, sd, undistortTable); // apply un-distortion based on distance + DepthImage32 d(sd.m_depthWidth, sd.m_depthHeight); + d.setInvalidValue(0.0f); + for (auto& v : d) { + unsigned int idx = v.y*sd.m_depthWidth + v.x; + v.value = (float)depth[idx] / sd.m_depthShift; + } + + // apply barell un-distortion to depth + d = (DepthImage32)Calibration::undistort(d, cd.depth_intrinsic, cd.depth_dist_coeff); + // align depth to color + d = Calibration::depthToColor(d, cd); + + // invalidate depth where we have no color + float scalarWidth = (float)(d.getWidth()-1) / (float)(sd.m_colorWidth-1); + float scalarHeight = (float)(d.getHeight()-1) / (float)(sd.m_colorHeight-1); + + for (auto& v : d) { + int x = math::round(v.x / scalarWidth); + int y = math::round(v.y / scalarHeight); + if (c(x,y) == vec3uc(0, 0, 0)) { + v.value = d.getInvalidValue(); + } + } + + // convert back to u16 + for (auto& v : d) { + unsigned int idx = v.y*(size_t)sd.m_depthWidth + v.x; + depth[idx] = math::round(v.value * sd.m_depthShift); + } + sd.replaceDepth(f, depth); + + std::free(depth); + } + std::cout << std::endl; + } + + D3D11GraphicsDevice* m_graphics; +}; \ No newline at end of file diff --git a/Calibrate/src/grid3d.cpp b/Calibrate/src/grid3d.cpp new file mode 100644 index 0000000..938838b --- /dev/null +++ b/Calibrate/src/grid3d.cpp @@ -0,0 +1,437 @@ +#include "stdafx.h" +#include "grid3d.h" + +Grid3D:: +Grid3D() +{ + m_xRes = -1; + m_yRes = -1; + m_zRes = -1; + m_maxDist = -1; + + m_data = nullptr; +} + +Grid3D:: +Grid3D(int xRes, int yRes, int zRes, float maxDist) +{ + m_xRes = xRes; + m_yRes = yRes; + m_zRes = zRes; + m_maxDist = maxDist; + + int nElements = NElements(); + m_data = new float[nElements]; + for (int i = 0; i < nElements; ++i) + { + m_data[i] = 0.0f; + } +} + +Grid3D:: +Grid3D(const Grid3D &other) +{ + m_xRes = other.XRes(); + m_yRes = other.YRes(); + m_zRes = other.ZRes(); + m_maxDist = other.MaxDist(); + + int nElements = NElements(); + m_data = new float[nElements]; + for (int i = 0; i < nElements; ++i) + { + m_data[i] = other.GetValue(i); + } +} + +Grid3D:: +Grid3D(const std::string &filename) +{ + ReadFile(filename); +} + +Grid3D:: +~Grid3D() +{ + if (m_data != nullptr) + { + delete [] m_data; + m_data = nullptr; + } + m_xRes = -1; + m_yRes = -1; + m_zRes = -1; + m_maxDist = -1; +} + +int Grid3D:: +ToIndex(int x, int y, int z) const +{ + return (z * m_xRes * m_yRes) + (y * m_xRes) + x; +} + +int Grid3D:: +XRes() const +{ + return m_xRes; +} + +int Grid3D:: +YRes() const +{ + return m_yRes; +} + +int Grid3D:: +ZRes() const +{ + return m_zRes; +} + +int Grid3D:: +NElements() const +{ + return m_xRes * m_yRes * m_zRes; +} + +float Grid3D:: +MaxDist() const +{ + return m_maxDist; +} + +float Grid3D:: +GetValue(int i) const +{ + assert(i >= 0 && i < NElements()); + return m_data[i]; +} + +float Grid3D:: +GetValue(int x, int y, int z) const +{ + assert(x >= 0 && x < m_xRes); + assert(y >= 0 && y < m_yRes); + assert(z >= 0 && z < m_zRes); + return m_data[ToIndex(x, y, z)]; +} + +float Grid3D:: +GetValue(float x, float y, float z) const +{ + assert(x >= 0 && x < m_xRes); + assert(y >= 0 && y < m_yRes); + assert(z >= 0 && z < m_zRes); + + int x1 = (int) x; + int y1 = (int) y; + int z1 = (int) z; + int x2 = x1 + 1; + int y2 = y1 + 1; + int z2 = z1 + 1; + if (x2 >= m_xRes) x2 = x1; + if (y2 >= m_yRes) y2 = y1; + if (z2 >= m_zRes) z2 = z1; + float dx = x - x1; + float dy = y - y1; + float dz = z - z1; + + float value = 0.0f; + value += GetValue(x1, y1, z1) * (1.0f - dx) * (1.0f - dy) * (1.0f - dz); + value += GetValue(x1, y1, z2) * (1.0f - dx) * (1.0f - dy) * dz; + value += GetValue(x1, y2, z1) * (1.0f - dx) * dy * (1.0f - dz); + value += GetValue(x1, y2, z2) * (1.0f - dx) * dy * dz; + + value += GetValue(x2, y1, z1) * dx * (1.0f - dy) * (1.0f - dz); + value += GetValue(x2, y1, z2) * dx * (1.0f - dy) * dz; + value += GetValue(x2, y2, z1) * dx * dy * (1.0f - dz); + value += GetValue(x2, y2, z2) * dx * dy * dz; + + return value; +} + +void Grid3D:: +SetValue(int i, float val) +{ + assert(i >= 0 && i < NElements()); + + m_data[i] = val; +} + +void Grid3D:: +SetValue(int x, int y, int z, float val) +{ + assert(x >= 0 && x < m_xRes); + assert(y >= 0 && y < m_yRes); + assert(z >= 0 && z < m_zRes); + + m_data[ToIndex(x, y, z)] = val; +} + +void Grid3D:: +Add(const float val) +{ + int nElements = NElements(); + for (int i = 0; i < nElements; ++i) + { + if (m_data[i] != GRID3D_UNKNOWN_VALUE) + m_data[i] += val; + } +} + +void Grid3D:: +Add(int x, int y, int z, float val) +{ + assert(x >= 0 && x < m_xRes); + assert(y >= 0 && y < m_yRes); + assert(z >= 0 && z < m_zRes); + int i = ToIndex(x, y, z); + if (m_data[i] != GRID3D_UNKNOWN_VALUE) + { + m_data[i] += val; + } +} + +void Grid3D:: +Add(const Grid3D &other) +{ + assert(other.XRes() == m_xRes); + assert(other.YRes() == m_yRes); + assert(other.ZRes() == m_zRes); + + int nElements = NElements(); + for (int i = 0; i < nElements; ++i) + { + if (m_data[i] != GRID3D_UNKNOWN_VALUE && + other.GetValue(i) != GRID3D_UNKNOWN_VALUE) + { + m_data[i] += other.GetValue(i); + } + } +} + +void Grid3D:: +Multiply(const float val) +{ + int nElements = NElements(); + for (int i = 0; i < nElements; ++i) + { + if (m_data[i] != GRID3D_UNKNOWN_VALUE) + { + m_data[i] *= val; + } + } +} + +void Grid3D:: +Multiply(int x, int y, int z, float val) +{ + assert(x >= 0 && x < m_xRes); + assert(y >= 0 && y < m_yRes); + assert(z >= 0 && z < m_zRes); + int i = ToIndex(x, y, z); + if (m_data[i] != GRID3D_UNKNOWN_VALUE) + { + m_data[i] *= val; + } +} + +void Grid3D:: +Multiply(const Grid3D &other) +{ + assert(other.XRes() == m_xRes); + assert(other.YRes() == m_yRes); + assert(other.ZRes() == m_zRes); + + int nElements = NElements(); + for (int i = 0; i < nElements; ++i) + { + if (m_data[i] != GRID3D_UNKNOWN_VALUE && + other.GetValue(i) != GRID3D_UNKNOWN_VALUE) + { + m_data[i] *= other.GetValue(i); + } + } +} + +void Grid3D:: +Divide(const float val) +{ + if (val == 0.0f) return; + + int nElements = NElements(); + for (int i = 0; i < nElements; ++i) + { + if (m_data[i] != GRID3D_UNKNOWN_VALUE) + { + m_data[i] /= val; + } + } +} + +void Grid3D:: +Divide(int x, int y, int z, float val) +{ + if (val == 0.0f) return; + assert(x >= 0 && x < m_xRes); + assert(y >= 0 && y < m_yRes); + assert(z >= 0 && z < m_zRes); + + int i = ToIndex(x, y, z); + if (m_data[i] != GRID3D_UNKNOWN_VALUE) + { + m_data[i] /= val; + } +} + +void Grid3D:: +Divide(const Grid3D &other) +{ + assert(other.XRes() == m_xRes); + assert(other.YRes() == m_yRes); + assert(other.ZRes() == m_zRes); + + int nElements = NElements(); + for (int i = 0; i < nElements; ++i) + { + if (other.GetValue(i) == 0.0f) // dont divide by zero. + { + m_data[i] = GRID3D_UNKNOWN_VALUE; + continue; + } + + if (m_data[i] != GRID3D_UNKNOWN_VALUE && + other.GetValue(i) != GRID3D_UNKNOWN_VALUE) + { + m_data[i] /= other.GetValue(i); + } + } +} + +void Grid3D:: +InvertElements() +{ + int nElements = NElements(); + for (int i = 0; i < nElements; ++i) + { + if (m_data[i] == 0.0f) // dont divide by zero. + { + m_data[i] = GRID3D_UNKNOWN_VALUE; + continue; + } + + if (m_data[i] != GRID3D_UNKNOWN_VALUE) + { + m_data[i] = 1.0f / m_data[i]; + } + } +} + +float Grid3D:: +Min() const +{ + float min = 1e9; + int nElements = NElements(); + for (int i = 0; i < nElements; ++i) + { + if (m_data[i] != GRID3D_UNKNOWN_VALUE && + m_data[i] < min) + { + min = m_data[i]; + } + } + return min; +} + +float Grid3D:: +Max() const +{ + float max = -1e9; + int nElements = NElements(); + for (int i = 0; i < nElements; ++i) + { + if (m_data[i] != GRID3D_UNKNOWN_VALUE && + m_data[i] > max) + { + max = m_data[i]; + } + } + return max; +} + +int Grid3D:: +ReadFile(const std::string &filename) +{ + // clear old data + if (m_data != nullptr) + { + delete [] m_data; + m_data = nullptr; + } + + // open file + FILE * fp = fopen(filename.c_str(), "rb"); + if (!fp) + { + printf("Could not open file %s for reading!\n", filename.c_str()); + fflush(stdout); + return 0; + } + + // read in header + int res[3]; + if (fread(res, sizeof(int), 3, fp) != 3) { + printf("Unable to read resolution from file %s\n", filename.c_str()); + return 0; + } + m_xRes = res[0]; + m_yRes = res[1]; + m_zRes = res[2]; + + if (fread(&m_maxDist, sizeof(float), 1, fp) != 1) { + printf("Unable to read max. distance from file %s\n", filename.c_str()); + return 0; + } + + // allocate and read data + int nElements = NElements(); + m_data = new float[nElements]; + if (fread(&(m_data[0]), sizeof(float), nElements, fp) != nElements) { + printf("Unable to read grid values from file %s\n", filename.c_str()); + return 0; + } + + return -1; +} + +int Grid3D:: +WriteFile(const std::string &filename) const +{ + FILE * fp = fopen(filename.c_str(), "wb"); + if (!fp) + { + printf("Could not open file %s for writing!\n", filename.c_str()); + return 0; + } + + if (fwrite(&m_xRes, sizeof(int), 3, fp) != 3) { + printf("Unable to write resolution to file %s\n", filename.c_str()); + return 0; + } + + if (fwrite(&m_maxDist, sizeof(float), 1, fp) != 1) { + printf("Unable to write maximum distance to file %s\n", filename.c_str()); + return 0; + } + + const int nElements = NElements(); + int test = (int)fwrite(&(m_data[0]), sizeof(float), nElements, fp); + if (test != nElements) { + printf("Unable to write grid values to file %s\n", filename.c_str()); + return 0; + } + + fclose(fp); + + return 1; +} \ No newline at end of file diff --git a/Calibrate/src/grid3d.h b/Calibrate/src/grid3d.h new file mode 100644 index 0000000..cd3309c --- /dev/null +++ b/Calibrate/src/grid3d.h @@ -0,0 +1,55 @@ +#pragma once + +#define GRID3D_UNKNOWN_VALUE -321 + +class Grid3D +{ +public: + Grid3D(); + Grid3D(int xRes, int yRes, int zRes, float maxDist); + Grid3D(const Grid3D &other); + Grid3D(const std::string &filename); + ~Grid3D(); + + int XRes() const; + int YRes() const; + int ZRes() const; + int NElements() const; + float MaxDist() const; + + float GetValue(int i) const; + float GetValue(int x, int y, int z) const; + float GetValue(float x, float y, float z) const; + void SetValue(int i, float val); + void SetValue(int x, int y, int z, float val); + + void Add(const float val); + void Add(int x, int y, int z, float val); + void Add(const Grid3D &other); + + void Multiply(const float val); + void Multiply(int x, int y, int z, float val); + void Multiply(const Grid3D &other); + + void Divide(const float val); + void Divide(int x, int y, int z, float val); + void Divide(const Grid3D &other); + + float Min() const; + float Max() const; + + void InvertElements(); + + int ReadFile(const std::string &filename); + int WriteFile(const std::string &filename) const; + +private: + int m_xRes; + int m_yRes; + int m_zRes; + float m_maxDist; + + float * m_data; + + int ToIndex(int x, int y, int z) const; +}; diff --git a/Calibrate/src/mLibInclude.h b/Calibrate/src/mLibInclude.h new file mode 100644 index 0000000..ddc5706 --- /dev/null +++ b/Calibrate/src/mLibInclude.h @@ -0,0 +1,20 @@ +#ifndef _MLIB_INCLUDE_H_ +#define _MLIB_INCLUDE_H_ + + +#include "mLibCore.h" +#include "mLibLodePNG.h" + +#include "mLibD3D11.h" +#include "mLibD3D11Font.h" + +#include "mLibFreeImage.h" +#include "mLibDepthCamera.h" + +#include "mLibZLib.h" + + + +using namespace ml; + +#endif \ No newline at end of file diff --git a/Calibrate/src/mLibSource.cpp b/Calibrate/src/mLibSource.cpp new file mode 100644 index 0000000..05f5606 --- /dev/null +++ b/Calibrate/src/mLibSource.cpp @@ -0,0 +1,8 @@ + +#include "stdafx.h" + +#include "mLibCore.cpp" +#include "mLibD3D11.cpp" +#include "mLibLodePNG.cpp" +#include "mLibDepthCamera.cpp" +#include "mLibZLib.cpp" \ No newline at end of file diff --git a/Calibrate/src/main.cpp b/Calibrate/src/main.cpp new file mode 100644 index 0000000..02f3cb7 --- /dev/null +++ b/Calibrate/src/main.cpp @@ -0,0 +1,80 @@ + +#include "stdafx.h" + +#include "main.h" +#include "calibration.h" +#include "aligner.h" + +std::string getCalibrationNameFromMap(const std::string& deviceCalibrationMapCsv, std::string scanDirectory) { + if (!util::directoryExists(scanDirectory)) throw MLIB_EXCEPTION(scanDirectory + " does not exist!"); + scanDirectory = util::replace(scanDirectory, '\\', '/'); + if (scanDirectory.back() == '/') scanDirectory.pop_back(); + const std::string scanMetaFile = scanDirectory + "/" + util::split(scanDirectory, '/').back() + ".txt"; + if (!util::fileExists(scanMetaFile)) return ""; + ParameterFile pf(scanMetaFile); std::string deviceId; + if (!pf.readParameter("deviceId", deviceId)) throw MLIB_EXCEPTION("no device id in meta file: " + scanMetaFile); + + std::unordered_map deviceIdToCalibrationName; + const std::string deviceIdHeader = "id", calibNameHeader = "calibration_name"; + unsigned int deviceIdIndex = (unsigned int)-1, calibNameIndex = (unsigned int)-1; + std::ifstream s(deviceCalibrationMapCsv); + //read header + std::string line; + if (!std::getline(s, line)) throw MLIB_EXCEPTION("failed to read device calibration map csv: " + deviceCalibrationMapCsv); + const auto headers = util::split(line, ','); + for (unsigned int i = 0; i < headers.size(); i++) { + if (headers[i] == deviceIdHeader) deviceIdIndex = i; + if (headers[i] == calibNameHeader) calibNameIndex = i; + } + if (deviceIdIndex == (unsigned int)-1 || calibNameIndex == (unsigned int)-1) throw MLIB_EXCEPTION("unable to find device id/calibration name in device calibration map cs file: " + deviceCalibrationMapCsv); + //find our device id + while (std::getline(s, line)) { + const auto elements = util::split(line, ',', true); //include empty strings + if (elements[deviceIdIndex] == deviceId) { + std::cout << "\tdevice id: " << deviceId << ", calibration name: " << elements[calibNameIndex] << std::endl; + return elements[calibNameIndex]; + } + } + s.close(); + + return ""; //nothing found +} + + +int main(int argc, char* argv[]) +{ + try { + if (argc == 5) { //converts a specific scan given by the command line arguments ( input_sens_file, output_sens_file, device_calibration_map, device_calibration_directory ) + const std::string inputSensFilename(argv[1]); + const std::string outputSensFilename(argv[2]); + const std::string deviceCalibrationMapFile(argv[3]); + std::string deviceCalibrationDir(argv[4]); + if (!(deviceCalibrationDir.back() == '/' || deviceCalibrationDir.back() == '\\')) deviceCalibrationDir.push_back('/'); + + const std::string calibrationName = getCalibrationNameFromMap(deviceCalibrationMapFile, util::directoryFromPath(inputSensFilename)); + if (!calibrationName.empty()) { + Calibration cb; + cb.calibrateScan(inputSensFilename, outputSensFilename, + deviceCalibrationDir + calibrationName + ".txt", deviceCalibrationDir + calibrationName + ".lut"); + } + else std::cout << "no calibration name found" << std::endl; + } + else { + throw MLIB_EXCEPTION("requires the input sens filepath, output sens filepath, parameter file, and input undistortion table as a command line arguments"); + } + } + catch (const std::exception& e) + { + MessageBoxA(NULL, e.what(), "Exception caught", MB_ICONERROR); + exit(EXIT_FAILURE); + } + catch (...) + { + MessageBoxA(NULL, "UNKNOWN EXCEPTION", "Exception caught", MB_ICONERROR); + exit(EXIT_FAILURE); + } + + return 0; +} + + diff --git a/Calibrate/src/main.h b/Calibrate/src/main.h new file mode 100644 index 0000000..9e593e5 --- /dev/null +++ b/Calibrate/src/main.h @@ -0,0 +1,9 @@ + +#include +#include +#include +#include +#include +#include + +#include "stdafx.h" diff --git a/Calibrate/src/processedFile.h b/Calibrate/src/processedFile.h new file mode 100644 index 0000000..3930691 --- /dev/null +++ b/Calibrate/src/processedFile.h @@ -0,0 +1,87 @@ +#pragma once + + +#include "mLibInclude.h" + +#define X_PROCESSED_FILE_STATE_FIELDS \ + X(bool, valid) \ + X(unsigned int, heapFreeCount) \ + X(unsigned int, numValidOptTransforms) \ + X(unsigned int, numTransforms) \ + X(bool, aligned) + + +#ifndef VAR_NAME +#define VAR_NAME(x) #x +#endif + +#define checkSizeArray(a, d)( (((sizeof a)/(sizeof a[0])) >= d)) + +class ProcessedFile +{ +public: + +#define X(type, name) type name; + X_PROCESSED_FILE_STATE_FIELDS +#undef X + + //! sets the parameter file and reads + void readMembers(const ParameterFile& parameterFile) { + m_ParameterFile = parameterFile; + readMembers(); + } + + //! reads all the members from the given parameter file (could be called for reloading) + void readMembers() { + aligned = false; + +#define X(type, name) \ + if (!m_ParameterFile.readParameter(std::string(#name), name) && std::string(#name) != "aligned") {MLIB_WARNING(std::string(#name).append(" ").append("uninitialized")); name = type();} + X_PROCESSED_FILE_STATE_FIELDS +#undef X + + m_bIsInitialized = true; + } + + template + std::string makeString(const T& in) { + std::string ret = std::to_string(in); + return ret; + } + template <> + std::string makeString(const bool& in) { + if (in == true) return "true"; + else return "false"; + } + + void saveToFile(const std::string& outFile) { + std::ofstream out(outFile); +#define X(type, name) \ + { out << #name << " = " << makeString(name) << std::endl; } + X_PROCESSED_FILE_STATE_FIELDS +#undef X + } + + void print() const { +#define X(type, name) \ + std::cout << #name " = " << name << std::endl; + X_PROCESSED_FILE_STATE_FIELDS +#undef X + } + + + //! constructor + ProcessedFile() { + m_bIsInitialized = false; + } + + //! destructor + ~ProcessedFile() { + } + + +private: + bool m_bIsInitialized; + ParameterFile m_ParameterFile; +}; + diff --git a/Calibrate/src/stdafx.cpp b/Calibrate/src/stdafx.cpp new file mode 100644 index 0000000..3f38672 --- /dev/null +++ b/Calibrate/src/stdafx.cpp @@ -0,0 +1,11 @@ +// stdafx.cpp : source file that includes just the standard includes +// VirtualScan.pch will be the pre-compiled header +// stdafx.obj will contain the pre-compiled type information + +#pragma warning (disable : 4996) + +#include "stdafx.h" + +#include "mLibSource.cpp" +// TODO: reference any additional headers you need in STDAFX.H +// and not in this file diff --git a/Calibrate/src/stdafx.h b/Calibrate/src/stdafx.h new file mode 100644 index 0000000..7c83918 --- /dev/null +++ b/Calibrate/src/stdafx.h @@ -0,0 +1,16 @@ +// stdafx.h : include file for standard system include files, +// or project specific include files that are used frequently, but +// are changed infrequently +// + +#pragma once + +#include +#include + +// TODO: reference additional headers your program requires here + +#include "WinSock2.h" +#include "windows.h" + +#include "mLibInclude.h" \ No newline at end of file diff --git a/CameraParameterEstimation/README.md b/CameraParameterEstimation/README.md index bb29ca5..e063244 100644 --- a/CameraParameterEstimation/README.md +++ b/CameraParameterEstimation/README.md @@ -3,7 +3,7 @@ Structured Light Camera Calibration This repository includes set of Matlab scripts and C++ programs used for calibration of structured light depth camera. Specifically, we have calibrated -a set of Structure Sensor Cameras from Occipital [link](http://http://occipital.com) +a set of Structure Sensor Cameras from [Occipital](http://http://occipital.com). We perform two types of calibration: 1. Depth-To-Color Calibration diff --git a/Converter/README.md b/Converter/README.md new file mode 100644 index 0000000..9367517 --- /dev/null +++ b/Converter/README.md @@ -0,0 +1,16 @@ +## Converter from ScannerApp capture to .sens data +=============================================== + +Converts raw capture data (`.h264`, `.depth`, `.txt`, `.imu`) output from the ScannerApp to `.sens` format. + +### Installation. +This code was developed under VS2013. + +Requirements: +- ffmpeg (assumes that `ffmpeg.exe` is at the relativel path `./ffmpeg/ffmpeg.exe`) +- our research library mLib, a git submodule in ../external/mLib +- mLib external libraries can be downloaded [here](https://www.dropbox.com/s/fve3uen5mzonidx/mLibExternal.zip?dl=0) + + +To run: +`converter.exe [path to directory of ScannerApp outputs] [name of ScannerApp output to convert]` diff --git a/Converter/converter.sln b/Converter/converter.sln new file mode 100644 index 0000000..ebfb91a --- /dev/null +++ b/Converter/converter.sln @@ -0,0 +1,22 @@ + +Microsoft Visual Studio Solution File, Format Version 12.00 +# Visual Studio 2013 +VisualStudioVersion = 12.0.40629.0 +MinimumVisualStudioVersion = 10.0.40219.1 +Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "converter", "converter.vcxproj", "{6DFC3959-985F-4B6D-96FC-D6A3CC41BDC3}" +EndProject +Global + GlobalSection(SolutionConfigurationPlatforms) = preSolution + Debug|x64 = Debug|x64 + Release|x64 = Release|x64 + EndGlobalSection + GlobalSection(ProjectConfigurationPlatforms) = postSolution + {6DFC3959-985F-4B6D-96FC-D6A3CC41BDC3}.Debug|x64.ActiveCfg = Debug|x64 + {6DFC3959-985F-4B6D-96FC-D6A3CC41BDC3}.Debug|x64.Build.0 = Debug|x64 + {6DFC3959-985F-4B6D-96FC-D6A3CC41BDC3}.Release|x64.ActiveCfg = Release|x64 + {6DFC3959-985F-4B6D-96FC-D6A3CC41BDC3}.Release|x64.Build.0 = Release|x64 + EndGlobalSection + GlobalSection(SolutionProperties) = preSolution + HideSolutionNode = FALSE + EndGlobalSection +EndGlobal diff --git a/Converter/converter.vcxproj b/Converter/converter.vcxproj new file mode 100644 index 0000000..0bbd410 --- /dev/null +++ b/Converter/converter.vcxproj @@ -0,0 +1,123 @@ + + + + + Debug + x64 + + + Release + x64 + + + + {6DFC3959-985F-4B6D-96FC-D6A3CC41BDC3} + Win32Proj + converter + + + + Application + true + v120 + Unicode + + + Application + false + v120 + true + Unicode + + + + + + + + + + + + + true + ./;./src/;../external/mLib/include/;../external/mLibExternal/include/;$(IncludePath) + ../external/mLibExternal/libsWindows/lib64/;$(LibraryPath) + + + false + ./;./src/;../external/mLib/include/;../external/mLibExternal/include/;$(IncludePath) + ../external/mLibExternal/libsWindows/lib64/;$(LibraryPath) + + + + Use + Level3 + Disabled + WIN32;_DEBUG;_CONSOLE;_LIB;%(PreprocessorDefinitions);_CRT_SECURE_NO_WARNINGS;NOMINMAX;_WINSOCK_DEPRECATED_NO_WARNINGS + true + stdafx.h + + + Console + true + zlib64.lib;FreeImage.lib;%(AdditionalDependencies) + + + + + Level3 + Use + MaxSpeed + true + true + WIN32;NDEBUG;_CONSOLE;_LIB;%(PreprocessorDefinitions);_CRT_SECURE_NO_WARNINGS;NOMINMAX;_WINSOCK_DEPRECATED_NO_WARNINGS + true + stdafx.h + + + Console + true + true + true + zlib64.lib;FreeImage.lib;%(AdditionalDependencies) + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Create + Create + + + + + + \ No newline at end of file diff --git a/Converter/converter.vcxproj.filters b/Converter/converter.vcxproj.filters new file mode 100644 index 0000000..d073cfc --- /dev/null +++ b/Converter/converter.vcxproj.filters @@ -0,0 +1,90 @@ + + + + + {67DA6AB6-F800-4c08-8B7A-83BB121AAD01} + rc;ico;cur;bmp;dlg;rc2;rct;bin;rgs;gif;jpg;jpeg;jpe;resx;tiff;tif;png;wav;mfcribbon-ms + + + {50fd99dc-5001-4643-abab-04f4c216b8a3} + + + {555583f0-31b9-4962-b5da-bb3133d6e4d9} + + + + + + + input + + + input + + + input + + + input + + + input + + + input + + + input + + + input + + + input + + + input + + + input + + + input + + + input + + + input + + + + sensorData + + + sensorData + + + sensorData + + + sensorData + + + sensorData + + + sensorData + + + sensorData + + + sensorData + + + + + + + \ No newline at end of file diff --git a/Converter/main.cpp b/Converter/main.cpp new file mode 100644 index 0000000..5dd1f21 --- /dev/null +++ b/Converter/main.cpp @@ -0,0 +1,241 @@ +// sensorFile.cpp : Defines the entry point for the console application. +// + +#include "stdafx.h" +#include + +#include "metaData.h" + + +//converts from seconds to microseconds +UINT64 timeToUINT64(double d) { + return (UINT64)(d*1000.0*1000.0); +} + + +void convertToSens(const std::string& baseFilename, ml::SensorData& sens) +{ + sens.free(); + + const std::string srcFileMeta = ml::util::removeExtensions(baseFilename) + ".txt"; + const std::string srcFileDepth = ml::util::removeExtensions(baseFilename) + ".depth"; + const std::string srcFileColor = ml::util::removeExtensions(baseFilename) + ".h264"; + const std::string srcFileIMU = ml::util::removeExtensions(baseFilename) + ".imu"; + + + if (!ml::util::fileExists(srcFileMeta)) throw MLIB_EXCEPTION("file not found " + srcFileMeta); + if (!ml::util::fileExists(srcFileDepth)) throw MLIB_EXCEPTION("file not found " + srcFileDepth); + if (!ml::util::fileExists(srcFileColor)) throw MLIB_EXCEPTION("file not found " + srcFileColor); + if (!ml::util::fileExists(srcFileIMU)) throw MLIB_EXCEPTION("file not found " + srcFileIMU); + + MetaData meta(srcFileMeta); + sens.initDefault( + meta.colorWidth, meta.colorHeight, + meta.depthWidth, meta.depthHeight, + meta.colorCalibration, + meta.depthCalibration, + ml::SensorData::COMPRESSION_TYPE_COLOR::TYPE_JPEG, + ml::SensorData::COMPRESSION_TYPE_DEPTH::TYPE_ZLIB_USHORT, + 1000.0f, + ml::SensorData::getName().StructureSensor + ); + + std::string execPath = ml::util::getExecutablePath(); + const std::string tmpDir = execPath + "/tmp1337/"; + ml::util::deleteDirectory(tmpDir); // in case we have a leftover directory + + const std::string baseNameColor = "frame-"; + std::string ffmpegPath = execPath + "../../ffmpeg/ffmpeg.exe"; + if (!ml::util::fileExists(ffmpegPath)) { + ffmpegPath = execPath + "./ffmpeg/ffmpeg.exe"; + } + + if (!ml::util::fileExists(ffmpegPath)) throw MLIB_EXCEPTION("could not find ffmpeg.exe in path: " + ffmpegPath); + std::string command = ffmpegPath + " -i " + srcFileColor + " " + tmpDir + baseNameColor + "%6d.color.png"; + std::cout << "running: " << command << std::endl; + ml::util::makeDirectory(tmpDir); + command = command + " > nul 2>&1"; //pipe output to dev/null + int res = system(command.c_str()); + + + std::ifstream inDepth(srcFileDepth, std::ios::binary); + + ml::Directory dir(tmpDir); + const unsigned int numColorFrames = (unsigned int)dir.getFiles().size(); + unsigned int numFrames = std::min(meta.numDepthFrames, meta.numColorFrames); + numFrames = std::min(numFrames, numColorFrames); + if (numFrames != meta.numDepthFrames || numFrames != meta.numColorFrames || numFrames != numColorFrames) { + MLIB_WARNING("frame counts are different:numColorImages(" + std::to_string(numColorFrames) + ") meta.numDepthImages(" + std::to_string(meta.numDepthFrames) + ") meta.numColorImages(" + std::to_string(meta.numColorFrames) + ")"); + } + + unsigned int frame = 0; + ml::SensorData::StringCounter scColor(tmpDir + "/" + baseNameColor, "color.png", 6); scColor.getNext(); + + while (frame < numFrames) { + assert(inDepth.good()); + + //ml::Timer t; + uint32_t byteSize; + inDepth.read((char*)&byteSize, sizeof(uint32_t)); + char* depthCompressed = new char[byteSize]; + inDepth.read((char*)depthCompressed, byteSize); + + const unsigned int depthWidth = meta.depthWidth; + const unsigned int depthHeight = meta.depthHeight; + + unsigned short* depth = (unsigned short*)std::malloc(depthWidth*depthHeight * 2); + + uplinksimple::decode((unsigned char*)depthCompressed, (unsigned int)byteSize, depthWidth*depthHeight, depth); + uplinksimple::shift2depth(depth, depthWidth*depthHeight); + + //check for invalid values + for (unsigned int i = 0; i < depthWidth*depthHeight; i++) { + if (depth[i] >= uplinksimple::shift2depth(0xffff)) { + depth[i] = 0; + } + } + + SAFE_DELETE(depthCompressed); + + std::string colorFile = scColor.getNext(); + ml::ColorImageR8G8B8 cImage; + ml::FreeImageWrapper::loadImage(colorFile, cImage); + + sens.addFrame(cImage.getData(), depth); + + std::free(depth); + std::cout << "\rframe " << frame << ": read " << byteSize << " [bytes] "; + frame++; + + //std::cout << t.getElapsedTimeMS() << " [ms]" << std::endl << std::endl; + } + + //read the rest of the depth frames + for (; frame < meta.numDepthFrames; frame++) { + assert(inDepth.good()); + uint32_t byteSize; + inDepth.read((char*)&byteSize, sizeof(uint32_t)); + char* depthCompressed = new char[byteSize]; + inDepth.read((char*)depthCompressed, byteSize); + SAFE_DELETE_ARRAY(depthCompressed); + } + + + for (unsigned int i = 0; i < meta.numDepthFrames; i++) { + //TODO read time stamps here + double timeStampDouble; + inDepth.read((char*)&timeStampDouble, sizeof(double)); + UINT64 timeStamp = timeToUINT64(timeStampDouble); + + if (i < (unsigned int)sens.m_frames.size()) { + sens.m_frames[i].setTimeStampColor(timeStamp); + sens.m_frames[i].setTimeStampDepth(timeStamp); + } + + //std::cout << timeStamp << std::endl; + //std::cout << std::setprecision(20) << timeStampDouble << std::endl; + } + + + std::ifstream inIMU(srcFileIMU, std::ios::binary); + for (unsigned int i = 0; i < meta.numIMUmeasurements; i++) { + ml::SensorData::IMUFrame f; + double timeStamp = 0.0; + inIMU.read((char*)&timeStamp, sizeof(double)); + + inIMU.read((char*)&f.rotationRate, sizeof(ml::vec3d)); + inIMU.read((char*)&f.acceleration, sizeof(ml::vec3d)); + inIMU.read((char*)&f.magneticField, sizeof(ml::vec3d)); + inIMU.read((char*)&f.attitude, sizeof(ml::vec3d)); + inIMU.read((char*)&f.gravity, sizeof(ml::vec3d)); + f.timeStamp = timeToUINT64(timeStamp); + + if (f.timeStamp == 0) { + std::cout << "invalid IMUFrame -> skipping" << std::endl; + continue; + } + sens.addIMUFrame(f); + } + inIMU.close(); + + std::cout << std::endl; + + //for (size_t i = 0; i < sens.m_frames.size(); i++) { + // const ml::SensorData::IMUFrame& f = sens.findClosestIMUFrame(i); + // std::cout << "frame time: " << sens.m_frames[i].getTimeStampColor() << "\t imu time: " << f.timeStamp << std::endl; + //} + + for (size_t i = 0; i < sens.m_frames.size(); i++) { + const ml::mat4f& t = sens.m_frames[i].getCameraToWorld(); + const ml::mat4f id = ml::mat4f::identity(); + for (unsigned int i = 0; i < 16; i++) { + if (t[i] != id[i]) throw MLIB_EXCEPTION("matrix needs to be the identity"); + } + + //std::cout << sens.m_frames[i].getCameraToWorld() << std::endl; + } + + ml::util::deleteDirectory(tmpDir); +} + +void processStagingFolder(std::string stagingFolder, const std::string& outSensFilename, bool forceOverwrite = false) +{ + stagingFolder = ml::util::replace(stagingFolder, "\\", "/"); //we assume forward slashes + + ml::Directory stagingFolderDir(stagingFolder); + + const std::vector tmp = ml::util::split(stagingFolder, "/"); + const std::string base = tmp.back(); + const std::string baseFile = stagingFolder + "/" + base; + + std::cout << "converting: " << baseFile << std::endl; + + //const std::string sensFile = baseFile + ".sens"; + if (ml::util::fileExists(outSensFilename)) { + std::cout << "sensFile already available: " << outSensFilename << "\n\t -> skipping folder" << std::endl; + return; + } + + ml::SensorData sd; + convertToSens(baseFile, sd); + sd.saveToFile(outSensFilename); + std::cout << sd << std::endl; + +} + + + +int main(int argc, char* argv[]) +{ +#if defined(DEBUG) | defined(_DEBUG) + _CrtSetDbgFlag(_CRTDBG_ALLOC_MEM_DF | _CRTDBG_LEAK_CHECK_DF); +#endif + //_CrtSetBreakAlloc(7545); + try { + + if (argc == 3) { //converts a specific scan given by the command line argument + std::string stagingFolder(argv[1]); + std::string outSensFilename(argv[2]); + std::cout << stagingFolder << std::endl; + std::cout << outSensFilename << std::endl; + processStagingFolder(stagingFolder, outSensFilename); + } + else { + throw MLIB_EXCEPTION("requires the path and output file as a command line arguments"); + } + } + catch (const std::exception& e) + { + MessageBoxA(NULL, e.what(), "Exception caught", MB_ICONERROR); + exit(EXIT_FAILURE); + } + catch (...) + { + MessageBoxA(NULL, "UNKNOWN EXCEPTION", "Exception caught", MB_ICONERROR); + exit(EXIT_FAILURE); + } + //std::cout << "" << std::endl; + //getchar(); + return 0; +} + diff --git a/Converter/src/mLibInclude.h b/Converter/src/mLibInclude.h new file mode 100644 index 0000000..6874251 --- /dev/null +++ b/Converter/src/mLibInclude.h @@ -0,0 +1,12 @@ + +// +// mLib includes +// + +#include "mLibCore.h" +//#include "mLibEigen.h" +#include "mLibLodePNG.h" +//#include "mLibZLib.h" +#include "mLibFreeImage.h" //This has to come after OpenMesh otherwise there is a crash +#include "mLibDepthCamera.h" + \ No newline at end of file diff --git a/Converter/src/mLibSource.cpp b/Converter/src/mLibSource.cpp new file mode 100644 index 0000000..826c78f --- /dev/null +++ b/Converter/src/mLibSource.cpp @@ -0,0 +1,7 @@ + +#include "stdafx.h" + +#include "mLibCore.cpp" +#include "mLibLodePNG.cpp" +//#include "mLibZLib.cpp" +#include "mLibDepthCamera.cpp" \ No newline at end of file diff --git a/Converter/src/metaData.h b/Converter/src/metaData.h new file mode 100644 index 0000000..6616c3f --- /dev/null +++ b/Converter/src/metaData.h @@ -0,0 +1,76 @@ + +#pragma once + +#include "stdafx.h" + +#ifndef VAR_NAME +#define VAR_NAME(x) #x +#endif + +struct MetaData { + + MetaData(const std::string& filename) { + loadFromFile(filename); + } + + void loadFromFile(const std::string& filename) { + ml::ParameterFile parameterFile(filename); + readParameter(parameterFile, std::string(VAR_NAME(numColorFrames)), numColorFrames); + readParameter(parameterFile, std::string(VAR_NAME(numDepthFrames)), numDepthFrames); + readParameter(parameterFile, std::string(VAR_NAME(numIMUmeasurements)), numIMUmeasurements); + readParameter(parameterFile, std::string(VAR_NAME(colorWidth)), colorWidth); + readParameter(parameterFile, std::string(VAR_NAME(colorHeight)), colorHeight); + readParameter(parameterFile, std::string(VAR_NAME(depthWidth)), depthWidth); + readParameter(parameterFile, std::string(VAR_NAME(depthHeight)), depthHeight); + + { + float fx_color, fy_color, mx_color, my_color; + readParameter(parameterFile, std::string(VAR_NAME(fx_color)), fx_color); + readParameter(parameterFile, std::string(VAR_NAME(fy_color)), fy_color); + readParameter(parameterFile, std::string(VAR_NAME(mx_color)), mx_color); + readParameter(parameterFile, std::string(VAR_NAME(my_color)), my_color); + colorCalibration = ml::SensorData::CalibrationData( + ml::mat4f( + fx_color, 0.0f, mx_color, 0.0f, + 0.0f, fy_color, my_color, 0.0f, + 0.0f, 0.0f, 1.0f, 0.0f, + 0.0f, 0.0f, 0.0f, 1.0f + ), + ml::mat4f::identity()); + } + { + float fx_depth, fy_depth, mx_depth, my_depth; + readParameter(parameterFile, std::string(VAR_NAME(fx_depth)), fx_depth); + readParameter(parameterFile, std::string(VAR_NAME(fy_depth)), fy_depth); + readParameter(parameterFile, std::string(VAR_NAME(mx_depth)), mx_depth); + readParameter(parameterFile, std::string(VAR_NAME(my_depth)), my_depth); + + ml::mat4f depthToColorExtrinsics = ml::mat4f::identity(); + if (readParameter(parameterFile, std::string(VAR_NAME(colorToDepthExtrinsics)), depthToColorExtrinsics)) depthToColorExtrinsics.invert(); + depthCalibration = ml::SensorData::CalibrationData( + ml::mat4f( + fx_depth, 0.0f, mx_depth, 0.0f, + 0.0f, fy_depth, my_depth, 0.0f, + 0.0f, 0.0f, 1.0f, 0.0f, + 0.0f, 0.0f, 0.0f, 1.0f + ), + depthToColorExtrinsics); + } + } + + template + bool readParameter(const ml::ParameterFile& pf, const std::string& varName, T& var) { + if (!pf.readParameter(varName, var)) { MLIB_WARNING(std::string(varName).append(" ").append("uninitialized")); return false; } + return true; + } + + unsigned int numColorFrames; + unsigned int numDepthFrames; + unsigned int numIMUmeasurements; + unsigned int colorWidth; + unsigned int colorHeight; + unsigned int depthWidth; + unsigned int depthHeight; + ml::SensorData::CalibrationData colorCalibration; + ml::SensorData::CalibrationData depthCalibration; +}; \ No newline at end of file diff --git a/Converter/src/targetver.h b/Converter/src/targetver.h new file mode 100644 index 0000000..87c0086 --- /dev/null +++ b/Converter/src/targetver.h @@ -0,0 +1,8 @@ +#pragma once + +// Including SDKDDKVer.h defines the highest available Windows platform. + +// If you wish to build your application for a previous Windows platform, include WinSDKVer.h and +// set the _WIN32_WINNT macro to the platform you wish to support before including SDKDDKVer.h. + +#include diff --git a/Converter/stdafx.cpp b/Converter/stdafx.cpp new file mode 100644 index 0000000..8a34e6e --- /dev/null +++ b/Converter/stdafx.cpp @@ -0,0 +1,10 @@ +// stdafx.cpp : source file that includes just the standard includes +// sensorFile.pch will be the pre-compiled header +// stdafx.obj will contain the pre-compiled type information + +#include "stdafx.h" + +// TODO: reference any additional headers you need in STDAFX.H +// and not in this file + +#include "mLibSource.cpp" diff --git a/Converter/stdafx.h b/Converter/stdafx.h new file mode 100644 index 0000000..99247d5 --- /dev/null +++ b/Converter/stdafx.h @@ -0,0 +1,18 @@ +// stdafx.h : include file for standard system include files, +// or project specific include files that are used frequently, but +// are changed infrequently +// + +#pragma once + +#include "targetver.h" + +#include +#include + +#include "WinSock2.h" +#include "windows.h" + +// TODO: reference additional headers your program requires here +#include "mLibInclude.h" + diff --git a/README.md b/README.md index 5e36e84..17eed44 100644 --- a/README.md +++ b/README.md @@ -4,9 +4,12 @@ ScanNet is an RGB-D video dataset containing 2.5 million views in more than 1500 ## ScanNet Data -If you would like to download the ScanNet data, please fill out an agreement to the [ScanNet Terms of Use](http://dovahkiin.stanford.edu/scannet-public/ScanNet_TOS.pdf) and send it to us at scannet@googlegroups.com. +If you would like to download the ScanNet data, fill out an agreement to the [ScanNet Terms of Use](http://kaldir.vc.cit.tum.de/scannet/ScanNet_TOS.pdf), using your institutional email addresses, and send it to us at scannet@googlegroups.com. + +If you have not received a response within a week, it is likely that your email is bouncing - please check this before sending repeat requests. Please do not reply to the noreply email - your email won't be seen. + +Please check the [changelog](http://www.scan-net.org/changelog) for updates to the data release. -If you have not received a response within a week, it is likely that your email is bouncing - please check this before sending repeat requests. ### Data Organization The data in ScanNet is organized by RGB-D sequence. Each sequence is stored under a directory with named `scene_`, or `scene%04d_%02d`, where each space corresponds to a unique location (0-indexed). The raw data captured during scanning, camera poses and surface mesh reconstructions, and annotation metadata are all stored together for the given sequence. The directory has the following structure: @@ -25,7 +28,7 @@ The data in ScanNet is organized by RGB-D sequence. Each sequence is stored unde |-- _vh_clean_2.0.010000.segs.json, _vh_clean.segs.json Over-segmentation of lo-res, hi-res meshes, respectively (referenced by aggregated semantic annotations) |-- _vh_clean_2.labels.ply - Visualization of aggregated semantic segmentation; colored by nyu40 labels (see img/legend; ply property 'label' denotes the ScanNet label id) + Visualization of aggregated semantic segmentation; colored by nyu40 labels (see img/legend; ply property 'label' denotes the nyu40 label id) |-- _2d-label.zip Raw 2d projections of aggregated annotation labels as 16-bit pngs with ScanNet label ids |-- _2d-instance.zip @@ -79,6 +82,7 @@ Compressed binary format with per-frame color, depth, camera pose and other data "segmentsFile": "..." // id of the *.segs.json segmentation file referenced } ``` +[BenchmarkScripts/util_3d.py](BenchmarkScripts/util_3d.py) gives examples to parsing the semantic instance information from the `*.segs.json`, `*.aggregation.json`, and `*_vh_clean_2.ply` mesh file, with example semantic segmentation visualization in [BenchmarkScripts/3d_helpers/visualize_labels_on_mesh.py](BenchmarkScripts/3d_helpers/visualize_labels_on_mesh.py). **2d annotation projections (`*_2d-label.zip`, `*_2d-instance.zip`, `*_2d-label-filt.zip`, `*_2d-instance-filt.zip`)**: Projection of 3d aggregated annotation of a scan into its RGB-D frames, according to the computed camera trajectory. @@ -122,6 +126,7 @@ We provide code for several scene understanding benchmarks on ScanNet: * 3D object retrieval * Semantic voxel labeling +Train/test splits are given at [Tasks/Benchmark](Tasks/Benchmark). Label mappings and trained models can be downloaded with the ScanNet data release. See [Tasks](Tasks). @@ -147,6 +152,6 @@ If you have any questions, please contact us at scannet@googlegroups.com ## Changelog ## License -The data is released under the [ScanNet Terms of Use](http://dovahkiin.stanford.edu/scannet-public/ScanNet_TOS.pdf), and the code is released under the MIT license. +The data is released under the [ScanNet Terms of Use](http://kaldir.vc.in.tum.de/scannet/ScanNet_TOS.pdf), and the code is released under the MIT license. Copyright (c) 2017 diff --git a/ScannerApp/Frameworks/GPUImage/framework/GPUImage.xcodeproj/project.pbxproj b/ScannerApp/Frameworks/GPUImage/framework/GPUImage.xcodeproj/project.pbxproj index 462f2f2..ac85190 100644 --- a/ScannerApp/Frameworks/GPUImage/framework/GPUImage.xcodeproj/project.pbxproj +++ b/ScannerApp/Frameworks/GPUImage/framework/GPUImage.xcodeproj/project.pbxproj @@ -1968,7 +1968,7 @@ isa = PBXProject; attributes = { LastTestingUpgradeCheck = 0510; - LastUpgradeCheck = 0700; + LastUpgradeCheck = 1000; ORGANIZATIONNAME = "Brad Larson"; TargetAttributes = { BCE209E41943F20C002FEED8 = { @@ -2401,6 +2401,7 @@ CLANG_CXX_LIBRARY = "libc++"; CLANG_WARN_DIRECT_OBJC_ISA_USAGE = YES_ERROR; CLANG_WARN_OBJC_ROOT_CLASS = YES_ERROR; + CODE_SIGN_IDENTITY = ""; "CODE_SIGN_IDENTITY[sdk=iphoneos*]" = "iPhone Developer"; CURRENT_PROJECT_VERSION = 1; DEFINES_MODULE = YES; @@ -2436,6 +2437,7 @@ CLANG_CXX_LIBRARY = "libc++"; CLANG_WARN_DIRECT_OBJC_ISA_USAGE = YES_ERROR; CLANG_WARN_OBJC_ROOT_CLASS = YES_ERROR; + CODE_SIGN_IDENTITY = ""; "CODE_SIGN_IDENTITY[sdk=iphoneos*]" = "iPhone Developer"; CURRENT_PROJECT_VERSION = 1; DEFINES_MODULE = YES; @@ -2466,11 +2468,21 @@ ALWAYS_SEARCH_USER_PATHS = NO; CLANG_ENABLE_MODULES = YES; CLANG_ENABLE_OBJC_ARC = YES; + CLANG_WARN_BLOCK_CAPTURE_AUTORELEASING = YES; CLANG_WARN_BOOL_CONVERSION = YES; + CLANG_WARN_COMMA = YES; CLANG_WARN_CONSTANT_CONVERSION = YES; + CLANG_WARN_DEPRECATED_OBJC_IMPLEMENTATIONS = YES; CLANG_WARN_EMPTY_BODY = YES; CLANG_WARN_ENUM_CONVERSION = YES; + CLANG_WARN_INFINITE_RECURSION = YES; CLANG_WARN_INT_CONVERSION = YES; + CLANG_WARN_NON_LITERAL_NULL_CONVERSION = YES; + CLANG_WARN_OBJC_IMPLICIT_RETAIN_SELF = YES; + CLANG_WARN_OBJC_LITERAL_CONVERSION = YES; + CLANG_WARN_RANGE_LOOP_ANALYSIS = YES; + CLANG_WARN_STRICT_PROTOTYPES = YES; + CLANG_WARN_SUSPICIOUS_MOVE = YES; CLANG_WARN_UNREACHABLE_CODE = YES; CLANG_WARN__DUPLICATE_METHOD_MATCH = YES; COPY_PHASE_STRIP = NO; @@ -2494,7 +2506,7 @@ GCC_WARN_UNINITIALIZED_AUTOS = YES; GCC_WARN_UNUSED_FUNCTION = YES; GCC_WARN_UNUSED_VARIABLE = YES; - IPHONEOS_DEPLOYMENT_TARGET = 4.3; + IPHONEOS_DEPLOYMENT_TARGET = 8.0; ONLY_ACTIVE_ARCH = YES; RUN_CLANG_STATIC_ANALYZER = YES; SDKROOT = iphoneos; @@ -2508,11 +2520,21 @@ ALWAYS_SEARCH_USER_PATHS = NO; CLANG_ENABLE_MODULES = YES; CLANG_ENABLE_OBJC_ARC = YES; + CLANG_WARN_BLOCK_CAPTURE_AUTORELEASING = YES; CLANG_WARN_BOOL_CONVERSION = YES; + CLANG_WARN_COMMA = YES; CLANG_WARN_CONSTANT_CONVERSION = YES; + CLANG_WARN_DEPRECATED_OBJC_IMPLEMENTATIONS = YES; CLANG_WARN_EMPTY_BODY = YES; CLANG_WARN_ENUM_CONVERSION = YES; + CLANG_WARN_INFINITE_RECURSION = YES; CLANG_WARN_INT_CONVERSION = YES; + CLANG_WARN_NON_LITERAL_NULL_CONVERSION = YES; + CLANG_WARN_OBJC_IMPLICIT_RETAIN_SELF = YES; + CLANG_WARN_OBJC_LITERAL_CONVERSION = YES; + CLANG_WARN_RANGE_LOOP_ANALYSIS = YES; + CLANG_WARN_STRICT_PROTOTYPES = YES; + CLANG_WARN_SUSPICIOUS_MOVE = YES; CLANG_WARN_UNREACHABLE_CODE = YES; CLANG_WARN__DUPLICATE_METHOD_MATCH = YES; COPY_PHASE_STRIP = YES; @@ -2528,7 +2550,7 @@ GCC_WARN_UNINITIALIZED_AUTOS = YES; GCC_WARN_UNUSED_FUNCTION = YES; GCC_WARN_UNUSED_VARIABLE = YES; - IPHONEOS_DEPLOYMENT_TARGET = 4.3; + IPHONEOS_DEPLOYMENT_TARGET = 8.0; RUN_CLANG_STATIC_ANALYZER = YES; SDKROOT = iphoneos; SYMROOT = "$(PROJECT_DIR)/../build"; diff --git a/ScannerApp/Frameworks/GPUImage/framework/GPUImage.xcodeproj/xcshareddata/xcschemes/Documentation.xcscheme b/ScannerApp/Frameworks/GPUImage/framework/GPUImage.xcodeproj/xcshareddata/xcschemes/Documentation.xcscheme index 880c931..9fe101f 100644 --- a/ScannerApp/Frameworks/GPUImage/framework/GPUImage.xcodeproj/xcshareddata/xcschemes/Documentation.xcscheme +++ b/ScannerApp/Frameworks/GPUImage/framework/GPUImage.xcodeproj/xcshareddata/xcschemes/Documentation.xcscheme @@ -1,6 +1,6 @@ - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/ScannerApp/Frameworks/GPUImage/framework/Source/GPUImage3x3TextureSamplingFilter.m b/ScannerApp/Frameworks/GPUImage/framework/Source/GPUImage3x3TextureSamplingFilter.m index 05c4d50..0550f04 100644 --- a/ScannerApp/Frameworks/GPUImage/framework/Source/GPUImage3x3TextureSamplingFilter.m +++ b/ScannerApp/Frameworks/GPUImage/framework/Source/GPUImage3x3TextureSamplingFilter.m @@ -84,16 +84,16 @@ - (void)setupFilterForSize:(CGSize)filterFrameSize; _texelHeight = 1.0 / filterFrameSize.height; runSynchronouslyOnVideoProcessingQueue(^{ - [GPUImageContext setActiveShaderProgram:filterProgram]; - if (GPUImageRotationSwapsWidthAndHeight(inputRotation)) + [GPUImageContext setActiveShaderProgram:self->filterProgram]; + if (GPUImageRotationSwapsWidthAndHeight(self->inputRotation)) { - glUniform1f(texelWidthUniform, _texelHeight); - glUniform1f(texelHeightUniform, _texelWidth); + glUniform1f(self->texelWidthUniform, self->_texelHeight); + glUniform1f(self->texelHeightUniform, self->_texelWidth); } else { - glUniform1f(texelWidthUniform, _texelWidth); - glUniform1f(texelHeightUniform, _texelHeight); + glUniform1f(self->texelWidthUniform, self->_texelWidth); + glUniform1f(self->texelHeightUniform, self->_texelHeight); } }); } diff --git a/ScannerApp/Frameworks/GPUImage/framework/Source/GPUImageAverageColor.m b/ScannerApp/Frameworks/GPUImage/framework/Source/GPUImageAverageColor.m index e2dd7e7..2135aaf 100644 --- a/ScannerApp/Frameworks/GPUImage/framework/Source/GPUImageAverageColor.m +++ b/ScannerApp/Frameworks/GPUImage/framework/Source/GPUImageAverageColor.m @@ -168,25 +168,25 @@ - (void)extractAverageColorAtFrameTime:(CMTime)frameTime; NSAssert(self.outputTextureOptions.internalFormat == GL_RGBA, @"The output texture internal format for this filter must be GL_RGBA."); NSAssert(self.outputTextureOptions.type == GL_UNSIGNED_BYTE, @"The type of the output texture of this filter must be GL_UNSIGNED_BYTE."); - NSUInteger totalNumberOfPixels = round(finalStageSize.width * finalStageSize.height); + NSUInteger totalNumberOfPixels = round(self->finalStageSize.width * self->finalStageSize.height); - if (rawImagePixels == NULL) + if (self->rawImagePixels == NULL) { - rawImagePixels = (GLubyte *)malloc(totalNumberOfPixels * 4); + self->rawImagePixels = (GLubyte *)malloc(totalNumberOfPixels * 4); } [GPUImageContext useImageProcessingContext]; - [outputFramebuffer activateFramebuffer]; - glReadPixels(0, 0, (int)finalStageSize.width, (int)finalStageSize.height, GL_RGBA, GL_UNSIGNED_BYTE, rawImagePixels); + [self->outputFramebuffer activateFramebuffer]; + glReadPixels(0, 0, (int)self->finalStageSize.width, (int)self->finalStageSize.height, GL_RGBA, GL_UNSIGNED_BYTE, self->rawImagePixels); NSUInteger redTotal = 0, greenTotal = 0, blueTotal = 0, alphaTotal = 0; NSUInteger byteIndex = 0; for (NSUInteger currentPixel = 0; currentPixel < totalNumberOfPixels; currentPixel++) { - redTotal += rawImagePixels[byteIndex++]; - greenTotal += rawImagePixels[byteIndex++]; - blueTotal += rawImagePixels[byteIndex++]; - alphaTotal += rawImagePixels[byteIndex++]; + redTotal += self->rawImagePixels[byteIndex++]; + greenTotal += self->rawImagePixels[byteIndex++]; + blueTotal += self->rawImagePixels[byteIndex++]; + alphaTotal += self->rawImagePixels[byteIndex++]; } CGFloat normalizedRedTotal = (CGFloat)redTotal / (CGFloat)totalNumberOfPixels / 255.0; @@ -194,9 +194,9 @@ - (void)extractAverageColorAtFrameTime:(CMTime)frameTime; CGFloat normalizedBlueTotal = (CGFloat)blueTotal / (CGFloat)totalNumberOfPixels / 255.0; CGFloat normalizedAlphaTotal = (CGFloat)alphaTotal / (CGFloat)totalNumberOfPixels / 255.0; - if (_colorAverageProcessingFinishedBlock != NULL) + if (self->_colorAverageProcessingFinishedBlock != NULL) { - _colorAverageProcessingFinishedBlock(normalizedRedTotal, normalizedGreenTotal, normalizedBlueTotal, normalizedAlphaTotal, frameTime); + self->_colorAverageProcessingFinishedBlock(normalizedRedTotal, normalizedGreenTotal, normalizedBlueTotal, normalizedAlphaTotal, frameTime); } }); } diff --git a/ScannerApp/Frameworks/GPUImage/framework/Source/GPUImageColorPackingFilter.m b/ScannerApp/Frameworks/GPUImage/framework/Source/GPUImageColorPackingFilter.m index 1a087ca..e98ba09 100644 --- a/ScannerApp/Frameworks/GPUImage/framework/Source/GPUImageColorPackingFilter.m +++ b/ScannerApp/Frameworks/GPUImage/framework/Source/GPUImageColorPackingFilter.m @@ -100,9 +100,9 @@ - (void)setupFilterForSize:(CGSize)filterFrameSize; texelHeight = 0.5 / inputTextureSize.height; runSynchronouslyOnVideoProcessingQueue(^{ - [GPUImageContext setActiveShaderProgram:filterProgram]; - glUniform1f(texelWidthUniform, texelWidth); - glUniform1f(texelHeightUniform, texelHeight); + [GPUImageContext setActiveShaderProgram:self->filterProgram]; + glUniform1f(self->texelWidthUniform, self->texelWidth); + glUniform1f(self->texelHeightUniform, self->texelHeight); }); } diff --git a/ScannerApp/Frameworks/GPUImage/framework/Source/GPUImageColourFASTSamplingOperation.m b/ScannerApp/Frameworks/GPUImage/framework/Source/GPUImageColourFASTSamplingOperation.m index fc67f47..b2107fe 100755 --- a/ScannerApp/Frameworks/GPUImage/framework/Source/GPUImageColourFASTSamplingOperation.m +++ b/ScannerApp/Frameworks/GPUImage/framework/Source/GPUImageColourFASTSamplingOperation.m @@ -167,16 +167,16 @@ - (void)setupFilterForSize:(CGSize)filterFrameSize; _texelHeight = 1.0 / filterFrameSize.height; runSynchronouslyOnVideoProcessingQueue(^{ - [GPUImageContext setActiveShaderProgram:filterProgram]; - if (GPUImageRotationSwapsWidthAndHeight(inputRotation)) + [GPUImageContext setActiveShaderProgram:self->filterProgram]; + if (GPUImageRotationSwapsWidthAndHeight(self->inputRotation)) { - glUniform1f(texelWidthUniform, _texelHeight); - glUniform1f(texelHeightUniform, _texelWidth); + glUniform1f(self->texelWidthUniform, self->_texelHeight); + glUniform1f(self->texelHeightUniform, self->_texelWidth); } else { - glUniform1f(texelWidthUniform, _texelWidth); - glUniform1f(texelHeightUniform, _texelHeight); + glUniform1f(self->texelWidthUniform, self->_texelWidth); + glUniform1f(self->texelHeightUniform, self->_texelHeight); } }); } @@ -201,4 +201,4 @@ - (void)setTexelHeight:(CGFloat)newValue; [self setFloat:_texelHeight forUniform:texelHeightUniform program:filterProgram]; } -@end \ No newline at end of file +@end diff --git a/ScannerApp/Frameworks/GPUImage/framework/Source/GPUImageCrosshairGenerator.m b/ScannerApp/Frameworks/GPUImage/framework/Source/GPUImageCrosshairGenerator.m index 9e2a29a..9c70fc3 100644 --- a/ScannerApp/Frameworks/GPUImage/framework/Source/GPUImageCrosshairGenerator.m +++ b/ScannerApp/Frameworks/GPUImage/framework/Source/GPUImageCrosshairGenerator.m @@ -71,8 +71,8 @@ - (id)init; } runSynchronouslyOnVideoProcessingQueue(^{ - crosshairWidthUniform = [filterProgram uniformIndex:@"crosshairWidth"]; - crosshairColorUniform = [filterProgram uniformIndex:@"crosshairColor"]; + self->crosshairWidthUniform = [self->filterProgram uniformIndex:@"crosshairWidth"]; + self->crosshairColorUniform = [self->filterProgram uniformIndex:@"crosshairColor"]; self.crosshairWidth = 5.0; [self setCrosshairColorRed:0.0 green:1.0 blue:0.0]; @@ -92,7 +92,7 @@ - (void)renderCrosshairsFromArray:(GLfloat *)crosshairCoordinates count:(NSUInte } runSynchronouslyOnVideoProcessingQueue(^{ - [GPUImageContext setActiveShaderProgram:filterProgram]; + [GPUImageContext setActiveShaderProgram:self->filterProgram]; #if TARGET_IPHONE_SIMULATOR || TARGET_OS_IPHONE #else @@ -100,13 +100,13 @@ - (void)renderCrosshairsFromArray:(GLfloat *)crosshairCoordinates count:(NSUInte glEnable(GL_VERTEX_PROGRAM_POINT_SIZE); #endif - outputFramebuffer = [[GPUImageContext sharedFramebufferCache] fetchFramebufferForSize:[self sizeOfFBO] textureOptions:self.outputTextureOptions onlyTexture:NO]; - [outputFramebuffer activateFramebuffer]; + self->outputFramebuffer = [[GPUImageContext sharedFramebufferCache] fetchFramebufferForSize:[self sizeOfFBO] textureOptions:self.outputTextureOptions onlyTexture:NO]; + [self->outputFramebuffer activateFramebuffer]; glClearColor(0.0, 0.0, 0.0, 0.0); glClear(GL_COLOR_BUFFER_BIT); - glVertexAttribPointer(filterPositionAttribute, 2, GL_FLOAT, 0, 0, crosshairCoordinates); + glVertexAttribPointer(self->filterPositionAttribute, 2, GL_FLOAT, 0, 0, crosshairCoordinates); glDrawArrays(GL_POINTS, 0, (GLsizei)numberOfCrosshairs); diff --git a/ScannerApp/Frameworks/GPUImage/framework/Source/GPUImageDirectionalNonMaximumSuppressionFilter.m b/ScannerApp/Frameworks/GPUImage/framework/Source/GPUImageDirectionalNonMaximumSuppressionFilter.m index b442f3a..d317b2f 100644 --- a/ScannerApp/Frameworks/GPUImage/framework/Source/GPUImageDirectionalNonMaximumSuppressionFilter.m +++ b/ScannerApp/Frameworks/GPUImage/framework/Source/GPUImageDirectionalNonMaximumSuppressionFilter.m @@ -96,9 +96,9 @@ - (void)setupFilterForSize:(CGSize)filterFrameSize; _texelHeight = 1.0 / filterFrameSize.height; runSynchronouslyOnVideoProcessingQueue(^{ - [GPUImageContext setActiveShaderProgram:filterProgram]; - glUniform1f(texelWidthUniform, _texelWidth); - glUniform1f(texelHeightUniform, _texelHeight); + [GPUImageContext setActiveShaderProgram:self->filterProgram]; + glUniform1f(self->texelWidthUniform, self->_texelWidth); + glUniform1f(self->texelHeightUniform, self->_texelHeight); }); } } diff --git a/ScannerApp/Frameworks/GPUImage/framework/Source/GPUImageFilter.m b/ScannerApp/Frameworks/GPUImage/framework/Source/GPUImageFilter.m index 406d707..96d8563 100755 --- a/ScannerApp/Frameworks/GPUImage/framework/Source/GPUImageFilter.m +++ b/ScannerApp/Frameworks/GPUImage/framework/Source/GPUImageFilter.m @@ -76,33 +76,33 @@ - (id)initWithVertexShaderFromString:(NSString *)vertexShaderString fragmentShad runSynchronouslyOnVideoProcessingQueue(^{ [GPUImageContext useImageProcessingContext]; - filterProgram = [[GPUImageContext sharedImageProcessingContext] programForVertexShaderString:vertexShaderString fragmentShaderString:fragmentShaderString]; + self->filterProgram = [[GPUImageContext sharedImageProcessingContext] programForVertexShaderString:vertexShaderString fragmentShaderString:fragmentShaderString]; - if (!filterProgram.initialized) + if (!self->filterProgram.initialized) { [self initializeAttributes]; - if (![filterProgram link]) + if (![self->filterProgram link]) { - NSString *progLog = [filterProgram programLog]; + NSString *progLog = [self->filterProgram programLog]; NSLog(@"Program link log: %@", progLog); - NSString *fragLog = [filterProgram fragmentShaderLog]; + NSString *fragLog = [self->filterProgram fragmentShaderLog]; NSLog(@"Fragment shader compile log: %@", fragLog); - NSString *vertLog = [filterProgram vertexShaderLog]; + NSString *vertLog = [self->filterProgram vertexShaderLog]; NSLog(@"Vertex shader compile log: %@", vertLog); - filterProgram = nil; + self->filterProgram = nil; NSAssert(NO, @"Filter shader link failed"); } } - filterPositionAttribute = [filterProgram attributeIndex:@"position"]; - filterTextureCoordinateAttribute = [filterProgram attributeIndex:@"inputTextureCoordinate"]; - filterInputTextureUniform = [filterProgram uniformIndex:@"inputImageTexture"]; // This does assume a name of "inputImageTexture" for the fragment shader + self->filterPositionAttribute = [self->filterProgram attributeIndex:@"position"]; + self->filterTextureCoordinateAttribute = [self->filterProgram attributeIndex:@"inputTextureCoordinate"]; + self->filterInputTextureUniform = [self->filterProgram uniformIndex:@"inputImageTexture"]; // This does assume a name of "inputImageTexture" for the fragment shader - [GPUImageContext setActiveShaderProgram:filterProgram]; + [GPUImageContext setActiveShaderProgram:self->filterProgram]; - glEnableVertexAttribArray(filterPositionAttribute); - glEnableVertexAttribArray(filterTextureCoordinateAttribute); + glEnableVertexAttribArray(self->filterPositionAttribute); + glEnableVertexAttribArray(self->filterTextureCoordinateAttribute); }); return self; diff --git a/ScannerApp/Frameworks/GPUImage/framework/Source/GPUImageFourInputFilter.m b/ScannerApp/Frameworks/GPUImage/framework/Source/GPUImageFourInputFilter.m index bc660a3..b82558d 100644 --- a/ScannerApp/Frameworks/GPUImage/framework/Source/GPUImageFourInputFilter.m +++ b/ScannerApp/Frameworks/GPUImage/framework/Source/GPUImageFourInputFilter.m @@ -58,10 +58,10 @@ - (id)initWithVertexShaderFromString:(NSString *)vertexShaderString fragmentShad runSynchronouslyOnVideoProcessingQueue(^{ [GPUImageContext useImageProcessingContext]; - filterFourthTextureCoordinateAttribute = [filterProgram attributeIndex:@"inputTextureCoordinate4"]; + self->filterFourthTextureCoordinateAttribute = [self->filterProgram attributeIndex:@"inputTextureCoordinate4"]; - filterInputTextureUniform4 = [filterProgram uniformIndex:@"inputImageTexture4"]; // This does assume a name of "inputImageTexture3" for the third input texture in the fragment shader - glEnableVertexAttribArray(filterFourthTextureCoordinateAttribute); + self->filterInputTextureUniform4 = [self->filterProgram uniformIndex:@"inputImageTexture4"]; // This does assume a name of "inputImageTexture3" for the third input texture in the fragment shader + glEnableVertexAttribArray(self->filterFourthTextureCoordinateAttribute); }); return self; diff --git a/ScannerApp/Frameworks/GPUImage/framework/Source/GPUImageFramebuffer.m b/ScannerApp/Frameworks/GPUImage/framework/Source/GPUImageFramebuffer.m index ea55c26..b71a2c9 100644 --- a/ScannerApp/Frameworks/GPUImage/framework/Source/GPUImageFramebuffer.m +++ b/ScannerApp/Frameworks/GPUImage/framework/Source/GPUImageFramebuffer.m @@ -51,7 +51,7 @@ - (id)initWithSize:(CGSize)framebufferSize textureOptions:(GPUTextureOptions)fbo runSynchronouslyOnVideoProcessingQueue(^{ [GPUImageContext useImageProcessingContext]; [self generateTexture]; - framebuffer = 0; + self->framebuffer = 0; }); } else @@ -133,8 +133,8 @@ - (void)generateFramebuffer; runSynchronouslyOnVideoProcessingQueue(^{ [GPUImageContext useImageProcessingContext]; - glGenFramebuffers(1, &framebuffer); - glBindFramebuffer(GL_FRAMEBUFFER, framebuffer); + glGenFramebuffers(1, &self->framebuffer); + glBindFramebuffer(GL_FRAMEBUFFER, self->framebuffer); // By default, all framebuffers on iOS 5.0+ devices are backed by texture caches, using one shared cache if ([GPUImageContext supportsFastTextureUpload]) @@ -149,23 +149,23 @@ - (void)generateFramebuffer; attrs = CFDictionaryCreateMutable(kCFAllocatorDefault, 1, &kCFTypeDictionaryKeyCallBacks, &kCFTypeDictionaryValueCallBacks); CFDictionarySetValue(attrs, kCVPixelBufferIOSurfacePropertiesKey, empty); - CVReturn err = CVPixelBufferCreate(kCFAllocatorDefault, (int)_size.width, (int)_size.height, kCVPixelFormatType_32BGRA, attrs, &renderTarget); + CVReturn err = CVPixelBufferCreate(kCFAllocatorDefault, (int)self->_size.width, (int)self->_size.height, kCVPixelFormatType_32BGRA, attrs, &self->renderTarget); if (err) { - NSLog(@"FBO size: %f, %f", _size.width, _size.height); + NSLog(@"FBO size: %f, %f", self->_size.width, self->_size.height); NSAssert(NO, @"Error at CVPixelBufferCreate %d", err); } - err = CVOpenGLESTextureCacheCreateTextureFromImage (kCFAllocatorDefault, coreVideoTextureCache, renderTarget, + err = CVOpenGLESTextureCacheCreateTextureFromImage (kCFAllocatorDefault, coreVideoTextureCache, self->renderTarget, NULL, // texture attributes GL_TEXTURE_2D, - _textureOptions.internalFormat, // opengl format - (int)_size.width, - (int)_size.height, - _textureOptions.format, // native iOS format - _textureOptions.type, + self->_textureOptions.internalFormat, // opengl format + (int)self.size.width, + (int)self.size.height, + self.textureOptions.format, // native iOS format + self.textureOptions.type, 0, - &renderTexture); + &self->renderTexture); if (err) { NSAssert(NO, @"Error at CVOpenGLESTextureCacheCreateTextureFromImage %d", err); @@ -174,22 +174,22 @@ - (void)generateFramebuffer; CFRelease(attrs); CFRelease(empty); - glBindTexture(CVOpenGLESTextureGetTarget(renderTexture), CVOpenGLESTextureGetName(renderTexture)); - _texture = CVOpenGLESTextureGetName(renderTexture); - glTexParameterf(GL_TEXTURE_2D, GL_TEXTURE_WRAP_S, _textureOptions.wrapS); - glTexParameterf(GL_TEXTURE_2D, GL_TEXTURE_WRAP_T, _textureOptions.wrapT); + glBindTexture(CVOpenGLESTextureGetTarget(self->renderTexture), CVOpenGLESTextureGetName(self->renderTexture)); + self->_texture = CVOpenGLESTextureGetName(self->renderTexture); + glTexParameterf(GL_TEXTURE_2D, GL_TEXTURE_WRAP_S, self->_textureOptions.wrapS); + glTexParameterf(GL_TEXTURE_2D, GL_TEXTURE_WRAP_T, self->_textureOptions.wrapT); - glFramebufferTexture2D(GL_FRAMEBUFFER, GL_COLOR_ATTACHMENT0, GL_TEXTURE_2D, CVOpenGLESTextureGetName(renderTexture), 0); + glFramebufferTexture2D(GL_FRAMEBUFFER, GL_COLOR_ATTACHMENT0, GL_TEXTURE_2D, CVOpenGLESTextureGetName(self->renderTexture), 0); #endif } else { [self generateTexture]; - glBindTexture(GL_TEXTURE_2D, _texture); + glBindTexture(GL_TEXTURE_2D, self->_texture); - glTexImage2D(GL_TEXTURE_2D, 0, _textureOptions.internalFormat, (int)_size.width, (int)_size.height, 0, _textureOptions.format, _textureOptions.type, 0); - glFramebufferTexture2D(GL_FRAMEBUFFER, GL_COLOR_ATTACHMENT0, GL_TEXTURE_2D, _texture, 0); + glTexImage2D(GL_TEXTURE_2D, 0, self->_textureOptions.internalFormat, (int)self->_size.width, (int)self->_size.height, 0, self->_textureOptions.format, self->_textureOptions.type, 0); + glFramebufferTexture2D(GL_FRAMEBUFFER, GL_COLOR_ATTACHMENT0, GL_TEXTURE_2D, self->_texture, 0); } #ifndef NS_BLOCK_ASSERTIONS @@ -206,32 +206,32 @@ - (void)destroyFramebuffer; runSynchronouslyOnVideoProcessingQueue(^{ [GPUImageContext useImageProcessingContext]; - if (framebuffer) + if (self->framebuffer) { - glDeleteFramebuffers(1, &framebuffer); - framebuffer = 0; + glDeleteFramebuffers(1, &self->framebuffer); + self->framebuffer = 0; } - if ([GPUImageContext supportsFastTextureUpload] && (!_missingFramebuffer)) + if ([GPUImageContext supportsFastTextureUpload] && (!self->_missingFramebuffer)) { #if TARGET_IPHONE_SIMULATOR || TARGET_OS_IPHONE - if (renderTarget) + if (self->renderTarget) { - CFRelease(renderTarget); - renderTarget = NULL; + CFRelease(self->renderTarget); + self->renderTarget = NULL; } - if (renderTexture) + if (self->renderTexture) { - CFRelease(renderTexture); - renderTexture = NULL; + CFRelease(self->renderTexture); + self->renderTexture = NULL; } #endif } else { - glDeleteTextures(1, &_texture); + glDeleteTextures(1, &self->_texture); } }); @@ -317,7 +317,7 @@ - (CGImageRef)newCGImageFromFramebufferContents; runSynchronouslyOnVideoProcessingQueue(^{ [GPUImageContext useImageProcessingContext]; - NSUInteger totalBytesForImage = (int)_size.width * (int)_size.height * 4; + NSUInteger totalBytesForImage = (int)self->_size.width * (int)self->_size.height * 4; // It appears that the width of a texture must be padded out to be a multiple of 8 (32 bytes) if reading from it using a texture cache GLubyte *rawImagePixels; @@ -326,13 +326,13 @@ - (CGImageRef)newCGImageFromFramebufferContents; if ([GPUImageContext supportsFastTextureUpload]) { #if TARGET_IPHONE_SIMULATOR || TARGET_OS_IPHONE - NSUInteger paddedWidthOfImage = CVPixelBufferGetBytesPerRow(renderTarget) / 4.0; - NSUInteger paddedBytesForImage = paddedWidthOfImage * (int)_size.height * 4; + NSUInteger paddedWidthOfImage = CVPixelBufferGetBytesPerRow(self->renderTarget) / 4.0; + NSUInteger paddedBytesForImage = paddedWidthOfImage * (int)self->_size.height * 4; glFinish(); - CFRetain(renderTarget); // I need to retain the pixel buffer here and release in the data source callback to prevent its bytes from being prematurely deallocated during a photo write operation + CFRetain(self->renderTarget); // I need to retain the pixel buffer here and release in the data source callback to prevent its bytes from being prematurely deallocated during a photo write operation [self lockForReading]; - rawImagePixels = (GLubyte *)CVPixelBufferGetBaseAddress(renderTarget); + rawImagePixels = (GLubyte *)CVPixelBufferGetBaseAddress(self->renderTarget); dataProvider = CGDataProviderCreateWithData((__bridge_retained void*)self, rawImagePixels, paddedBytesForImage, dataProviderUnlockCallback); [[GPUImageContext sharedFramebufferCache] addFramebufferToActiveImageCaptureList:self]; // In case the framebuffer is swapped out on the filter, need to have a strong reference to it somewhere for it to hang on while the image is in existence #else @@ -342,7 +342,7 @@ - (CGImageRef)newCGImageFromFramebufferContents; { [self activateFramebuffer]; rawImagePixels = (GLubyte *)malloc(totalBytesForImage); - glReadPixels(0, 0, (int)_size.width, (int)_size.height, GL_RGBA, GL_UNSIGNED_BYTE, rawImagePixels); + glReadPixels(0, 0, (int)self->_size.width, (int)self->_size.height, GL_RGBA, GL_UNSIGNED_BYTE, rawImagePixels); dataProvider = CGDataProviderCreateWithData(NULL, rawImagePixels, totalBytesForImage, dataProviderReleaseCallback); [self unlock]; // Don't need to keep this around anymore } @@ -352,13 +352,13 @@ - (CGImageRef)newCGImageFromFramebufferContents; if ([GPUImageContext supportsFastTextureUpload]) { #if TARGET_IPHONE_SIMULATOR || TARGET_OS_IPHONE - cgImageFromBytes = CGImageCreate((int)_size.width, (int)_size.height, 8, 32, CVPixelBufferGetBytesPerRow(renderTarget), defaultRGBColorSpace, kCGBitmapByteOrder32Little | kCGImageAlphaPremultipliedFirst, dataProvider, NULL, NO, kCGRenderingIntentDefault); + cgImageFromBytes = CGImageCreate((int)self->_size.width, (int)self->_size.height, 8, 32, CVPixelBufferGetBytesPerRow(self->renderTarget), defaultRGBColorSpace, kCGBitmapByteOrder32Little | kCGImageAlphaPremultipliedFirst, dataProvider, NULL, NO, kCGRenderingIntentDefault); #else #endif } else { - cgImageFromBytes = CGImageCreate((int)_size.width, (int)_size.height, 8, 32, 4 * (int)_size.width, defaultRGBColorSpace, kCGBitmapByteOrderDefault | kCGImageAlphaLast, dataProvider, NULL, NO, kCGRenderingIntentDefault); + cgImageFromBytes = CGImageCreate((int)self->_size.width, (int)self->_size.height, 8, 32, 4 * (int)self->_size.width, defaultRGBColorSpace, kCGBitmapByteOrderDefault | kCGImageAlphaLast, dataProvider, NULL, NO, kCGRenderingIntentDefault); } // Capture image with current device orientation diff --git a/ScannerApp/Frameworks/GPUImage/framework/Source/GPUImageFramebufferCache.m b/ScannerApp/Frameworks/GPUImage/framework/Source/GPUImageFramebufferCache.m index 53faf2c..aa6e4b1 100644 --- a/ScannerApp/Frameworks/GPUImage/framework/Source/GPUImageFramebufferCache.m +++ b/ScannerApp/Frameworks/GPUImage/framework/Source/GPUImageFramebufferCache.m @@ -84,7 +84,7 @@ - (GPUImageFramebuffer *)fetchFramebufferForSize:(CGSize)framebufferSize texture // dispatch_sync(framebufferCacheQueue, ^{ runSynchronouslyOnVideoProcessingQueue(^{ NSString *lookupHash = [self hashForSize:framebufferSize textureOptions:textureOptions onlyTexture:onlyTexture]; - NSNumber *numberOfMatchingTexturesInCache = [framebufferTypeCounts objectForKey:lookupHash]; + NSNumber *numberOfMatchingTexturesInCache = [self->framebufferTypeCounts objectForKey:lookupHash]; NSInteger numberOfMatchingTextures = [numberOfMatchingTexturesInCache integerValue]; if ([numberOfMatchingTexturesInCache integerValue] < 1) @@ -99,19 +99,19 @@ - (GPUImageFramebuffer *)fetchFramebufferForSize:(CGSize)framebufferSize texture while ((framebufferFromCache == nil) && (currentTextureID >= 0)) { NSString *textureHash = [NSString stringWithFormat:@"%@-%ld", lookupHash, (long)currentTextureID]; - framebufferFromCache = [framebufferCache objectForKey:textureHash]; + framebufferFromCache = [self->framebufferCache objectForKey:textureHash]; // Test the values in the cache first, to see if they got invalidated behind our back if (framebufferFromCache != nil) { // Withdraw this from the cache while it's in use - [framebufferCache removeObjectForKey:textureHash]; + [self->framebufferCache removeObjectForKey:textureHash]; } currentTextureID--; } currentTextureID++; - [framebufferTypeCounts setObject:[NSNumber numberWithInteger:currentTextureID] forKey:lookupHash]; + [self->framebufferTypeCounts setObject:[NSNumber numberWithInteger:currentTextureID] forKey:lookupHash]; if (framebufferFromCache == nil) { @@ -147,14 +147,14 @@ - (void)returnFramebufferToCache:(GPUImageFramebuffer *)framebuffer; CGSize framebufferSize = framebuffer.size; GPUTextureOptions framebufferTextureOptions = framebuffer.textureOptions; NSString *lookupHash = [self hashForSize:framebufferSize textureOptions:framebufferTextureOptions onlyTexture:framebuffer.missingFramebuffer]; - NSNumber *numberOfMatchingTexturesInCache = [framebufferTypeCounts objectForKey:lookupHash]; + NSNumber *numberOfMatchingTexturesInCache = [self->framebufferTypeCounts objectForKey:lookupHash]; NSInteger numberOfMatchingTextures = [numberOfMatchingTexturesInCache integerValue]; NSString *textureHash = [NSString stringWithFormat:@"%@-%ld", lookupHash, (long)numberOfMatchingTextures]; // [framebufferCache setObject:framebuffer forKey:textureHash cost:round(framebufferSize.width * framebufferSize.height * 4.0)]; - [framebufferCache setObject:framebuffer forKey:textureHash]; - [framebufferTypeCounts setObject:[NSNumber numberWithInteger:(numberOfMatchingTextures + 1)] forKey:lookupHash]; + [self->framebufferCache setObject:framebuffer forKey:textureHash]; + [self->framebufferTypeCounts setObject:[NSNumber numberWithInteger:(numberOfMatchingTextures + 1)] forKey:lookupHash]; }); } @@ -162,8 +162,8 @@ - (void)purgeAllUnassignedFramebuffers; { runAsynchronouslyOnVideoProcessingQueue(^{ // dispatch_async(framebufferCacheQueue, ^{ - [framebufferCache removeAllObjects]; - [framebufferTypeCounts removeAllObjects]; + [self->framebufferCache removeAllObjects]; + [self->framebufferTypeCounts removeAllObjects]; #if TARGET_IPHONE_SIMULATOR || TARGET_OS_IPHONE CVOpenGLESTextureCacheFlush([[GPUImageContext sharedImageProcessingContext] coreVideoTextureCache], 0); #else @@ -175,7 +175,7 @@ - (void)addFramebufferToActiveImageCaptureList:(GPUImageFramebuffer *)framebuffe { runAsynchronouslyOnVideoProcessingQueue(^{ // dispatch_async(framebufferCacheQueue, ^{ - [activeImageCaptureList addObject:framebuffer]; + [self->activeImageCaptureList addObject:framebuffer]; }); } @@ -183,7 +183,7 @@ - (void)removeFramebufferFromActiveImageCaptureList:(GPUImageFramebuffer *)frame { runAsynchronouslyOnVideoProcessingQueue(^{ // dispatch_async(framebufferCacheQueue, ^{ - [activeImageCaptureList removeObject:framebuffer]; + [self->activeImageCaptureList removeObject:framebuffer]; }); } diff --git a/ScannerApp/Frameworks/GPUImage/framework/Source/GPUImageGaussianBlurFilter.m b/ScannerApp/Frameworks/GPUImage/framework/Source/GPUImageGaussianBlurFilter.m index ec99352..65e25f1 100755 --- a/ScannerApp/Frameworks/GPUImage/framework/Source/GPUImageGaussianBlurFilter.m +++ b/ScannerApp/Frameworks/GPUImage/framework/Source/GPUImageGaussianBlurFilter.m @@ -383,64 +383,64 @@ - (void)switchToVertexShader:(NSString *)newVertexShader fragmentShader:(NSStrin runSynchronouslyOnVideoProcessingQueue(^{ [GPUImageContext useImageProcessingContext]; - filterProgram = [[GPUImageContext sharedImageProcessingContext] programForVertexShaderString:newVertexShader fragmentShaderString:newFragmentShader]; + self->filterProgram = [[GPUImageContext sharedImageProcessingContext] programForVertexShaderString:newVertexShader fragmentShaderString:newFragmentShader]; - if (!filterProgram.initialized) + if (!self->filterProgram.initialized) { [self initializeAttributes]; - if (![filterProgram link]) + if (![self->filterProgram link]) { - NSString *progLog = [filterProgram programLog]; + NSString *progLog = [self->filterProgram programLog]; NSLog(@"Program link log: %@", progLog); - NSString *fragLog = [filterProgram fragmentShaderLog]; + NSString *fragLog = [self->filterProgram fragmentShaderLog]; NSLog(@"Fragment shader compile log: %@", fragLog); - NSString *vertLog = [filterProgram vertexShaderLog]; + NSString *vertLog = [self->filterProgram vertexShaderLog]; NSLog(@"Vertex shader compile log: %@", vertLog); - filterProgram = nil; + self->filterProgram = nil; NSAssert(NO, @"Filter shader link failed"); } } - filterPositionAttribute = [filterProgram attributeIndex:@"position"]; - filterTextureCoordinateAttribute = [filterProgram attributeIndex:@"inputTextureCoordinate"]; - filterInputTextureUniform = [filterProgram uniformIndex:@"inputImageTexture"]; // This does assume a name of "inputImageTexture" for the fragment shader - verticalPassTexelWidthOffsetUniform = [filterProgram uniformIndex:@"texelWidthOffset"]; - verticalPassTexelHeightOffsetUniform = [filterProgram uniformIndex:@"texelHeightOffset"]; - [GPUImageContext setActiveShaderProgram:filterProgram]; + self->filterPositionAttribute = [self->filterProgram attributeIndex:@"position"]; + self->filterTextureCoordinateAttribute = [self->filterProgram attributeIndex:@"inputTextureCoordinate"]; + self->filterInputTextureUniform = [self->filterProgram uniformIndex:@"inputImageTexture"]; // This does assume a name of "inputImageTexture" for the fragment shader + self->verticalPassTexelWidthOffsetUniform = [self->filterProgram uniformIndex:@"texelWidthOffset"]; + self->verticalPassTexelHeightOffsetUniform = [self->filterProgram uniformIndex:@"texelHeightOffset"]; + [GPUImageContext setActiveShaderProgram:self->filterProgram]; - glEnableVertexAttribArray(filterPositionAttribute); - glEnableVertexAttribArray(filterTextureCoordinateAttribute); + glEnableVertexAttribArray(self->filterPositionAttribute); + glEnableVertexAttribArray(self->filterTextureCoordinateAttribute); - secondFilterProgram = [[GPUImageContext sharedImageProcessingContext] programForVertexShaderString:newVertexShader fragmentShaderString:newFragmentShader]; + self->secondFilterProgram = [[GPUImageContext sharedImageProcessingContext] programForVertexShaderString:newVertexShader fragmentShaderString:newFragmentShader]; - if (!secondFilterProgram.initialized) + if (!self->secondFilterProgram.initialized) { [self initializeSecondaryAttributes]; - if (![secondFilterProgram link]) + if (![self->secondFilterProgram link]) { - NSString *progLog = [secondFilterProgram programLog]; + NSString *progLog = [self->secondFilterProgram programLog]; NSLog(@"Program link log: %@", progLog); - NSString *fragLog = [secondFilterProgram fragmentShaderLog]; + NSString *fragLog = [self->secondFilterProgram fragmentShaderLog]; NSLog(@"Fragment shader compile log: %@", fragLog); - NSString *vertLog = [secondFilterProgram vertexShaderLog]; + NSString *vertLog = [self->secondFilterProgram vertexShaderLog]; NSLog(@"Vertex shader compile log: %@", vertLog); - secondFilterProgram = nil; + self->secondFilterProgram = nil; NSAssert(NO, @"Filter shader link failed"); } } - secondFilterPositionAttribute = [secondFilterProgram attributeIndex:@"position"]; - secondFilterTextureCoordinateAttribute = [secondFilterProgram attributeIndex:@"inputTextureCoordinate"]; - secondFilterInputTextureUniform = [secondFilterProgram uniformIndex:@"inputImageTexture"]; // This does assume a name of "inputImageTexture" for the fragment shader - secondFilterInputTextureUniform2 = [secondFilterProgram uniformIndex:@"inputImageTexture2"]; // This does assume a name of "inputImageTexture2" for second input texture in the fragment shader - horizontalPassTexelWidthOffsetUniform = [secondFilterProgram uniformIndex:@"texelWidthOffset"]; - horizontalPassTexelHeightOffsetUniform = [secondFilterProgram uniformIndex:@"texelHeightOffset"]; - [GPUImageContext setActiveShaderProgram:secondFilterProgram]; - - glEnableVertexAttribArray(secondFilterPositionAttribute); - glEnableVertexAttribArray(secondFilterTextureCoordinateAttribute); + self->secondFilterPositionAttribute = [self->secondFilterProgram attributeIndex:@"position"]; + self->secondFilterTextureCoordinateAttribute = [self->secondFilterProgram attributeIndex:@"inputTextureCoordinate"]; + self->secondFilterInputTextureUniform = [self->secondFilterProgram uniformIndex:@"inputImageTexture"]; // This does assume a name of "inputImageTexture" for the fragment shader + self->secondFilterInputTextureUniform2 = [self->secondFilterProgram uniformIndex:@"inputImageTexture2"]; // This does assume a name of "inputImageTexture2" for second input texture in the fragment shader + self->horizontalPassTexelWidthOffsetUniform = [self->secondFilterProgram uniformIndex:@"texelWidthOffset"]; + self->horizontalPassTexelHeightOffsetUniform = [self->secondFilterProgram uniformIndex:@"texelHeightOffset"]; + [GPUImageContext setActiveShaderProgram:self->secondFilterProgram]; + + glEnableVertexAttribArray(self->secondFilterPositionAttribute); + glEnableVertexAttribArray(self->secondFilterTextureCoordinateAttribute); [self setupFilterForSize:[self sizeOfFBO]]; glFinish(); diff --git a/ScannerApp/Frameworks/GPUImage/framework/Source/GPUImageHistogramFilter.m b/ScannerApp/Frameworks/GPUImage/framework/Source/GPUImageHistogramFilter.m index bb7acce..9664144 100755 --- a/ScannerApp/Frameworks/GPUImage/framework/Source/GPUImageHistogramFilter.m +++ b/ScannerApp/Frameworks/GPUImage/framework/Source/GPUImageHistogramFilter.m @@ -145,50 +145,50 @@ - (id)initWithHistogramType:(GPUImageHistogramType)newHistogramType; runSynchronouslyOnVideoProcessingQueue(^{ [GPUImageContext useImageProcessingContext]; - secondFilterProgram = [[GPUImageContext sharedImageProcessingContext] programForVertexShaderString:kGPUImageGreenHistogramSamplingVertexShaderString fragmentShaderString:kGPUImageHistogramAccumulationFragmentShaderString]; - thirdFilterProgram = [[GPUImageContext sharedImageProcessingContext] programForVertexShaderString:kGPUImageBlueHistogramSamplingVertexShaderString fragmentShaderString:kGPUImageHistogramAccumulationFragmentShaderString]; + self->secondFilterProgram = [[GPUImageContext sharedImageProcessingContext] programForVertexShaderString:kGPUImageGreenHistogramSamplingVertexShaderString fragmentShaderString:kGPUImageHistogramAccumulationFragmentShaderString]; + self->thirdFilterProgram = [[GPUImageContext sharedImageProcessingContext] programForVertexShaderString:kGPUImageBlueHistogramSamplingVertexShaderString fragmentShaderString:kGPUImageHistogramAccumulationFragmentShaderString]; - if (!secondFilterProgram.initialized) + if (!self->secondFilterProgram.initialized) { [self initializeSecondaryAttributes]; - if (![secondFilterProgram link]) + if (![self->secondFilterProgram link]) { - NSString *progLog = [secondFilterProgram programLog]; + NSString *progLog = [self->secondFilterProgram programLog]; NSLog(@"Program link log: %@", progLog); - NSString *fragLog = [secondFilterProgram fragmentShaderLog]; + NSString *fragLog = [self->secondFilterProgram fragmentShaderLog]; NSLog(@"Fragment shader compile log: %@", fragLog); - NSString *vertLog = [secondFilterProgram vertexShaderLog]; + NSString *vertLog = [self->secondFilterProgram vertexShaderLog]; NSLog(@"Vertex shader compile log: %@", vertLog); - filterProgram = nil; + self->filterProgram = nil; NSAssert(NO, @"Filter shader link failed"); } - [GPUImageContext setActiveShaderProgram:secondFilterProgram]; + [GPUImageContext setActiveShaderProgram:self->secondFilterProgram]; - glEnableVertexAttribArray(secondFilterPositionAttribute); + glEnableVertexAttribArray(self->secondFilterPositionAttribute); - if (![thirdFilterProgram link]) + if (![self->thirdFilterProgram link]) { - NSString *progLog = [secondFilterProgram programLog]; + NSString *progLog = [self->secondFilterProgram programLog]; NSLog(@"Program link log: %@", progLog); - NSString *fragLog = [secondFilterProgram fragmentShaderLog]; + NSString *fragLog = [self->secondFilterProgram fragmentShaderLog]; NSLog(@"Fragment shader compile log: %@", fragLog); - NSString *vertLog = [secondFilterProgram vertexShaderLog]; + NSString *vertLog = [self->secondFilterProgram vertexShaderLog]; NSLog(@"Vertex shader compile log: %@", vertLog); - filterProgram = nil; + self->filterProgram = nil; NSAssert(NO, @"Filter shader link failed"); } } - secondFilterPositionAttribute = [secondFilterProgram attributeIndex:@"position"]; + self->secondFilterPositionAttribute = [self->secondFilterProgram attributeIndex:@"position"]; - thirdFilterPositionAttribute = [thirdFilterProgram attributeIndex:@"position"]; - [GPUImageContext setActiveShaderProgram:thirdFilterProgram]; + self->thirdFilterPositionAttribute = [self->thirdFilterProgram attributeIndex:@"position"]; + [GPUImageContext setActiveShaderProgram:self->thirdFilterProgram]; - glEnableVertexAttribArray(thirdFilterPositionAttribute); + glEnableVertexAttribArray(self->thirdFilterPositionAttribute); }); }; break; } diff --git a/ScannerApp/Frameworks/GPUImage/framework/Source/GPUImageLanczosResamplingFilter.m b/ScannerApp/Frameworks/GPUImage/framework/Source/GPUImageLanczosResamplingFilter.m index a655f48..506359d 100644 --- a/ScannerApp/Frameworks/GPUImage/framework/Source/GPUImageLanczosResamplingFilter.m +++ b/ScannerApp/Frameworks/GPUImage/framework/Source/GPUImageLanczosResamplingFilter.m @@ -145,19 +145,19 @@ - (void)setupFilterForSize:(CGSize)filterFrameSize; { runSynchronouslyOnVideoProcessingQueue(^{ // The first pass through the framebuffer may rotate the inbound image, so need to account for that by changing up the kernel ordering for that pass - if (GPUImageRotationSwapsWidthAndHeight(inputRotation)) + if (GPUImageRotationSwapsWidthAndHeight(self->inputRotation)) { - verticalPassTexelWidthOffset = 1.0 / _originalImageSize.height; - verticalPassTexelHeightOffset = 0.0; + self->verticalPassTexelWidthOffset = 1.0 / self->_originalImageSize.height; + self->verticalPassTexelHeightOffset = 0.0; } else { - verticalPassTexelWidthOffset = 0.0; - verticalPassTexelHeightOffset = 1.0 / _originalImageSize.height; + self->verticalPassTexelWidthOffset = 0.0; + self->verticalPassTexelHeightOffset = 1.0 / self->_originalImageSize.height; } - horizontalPassTexelWidthOffset = 1.0 / _originalImageSize.width; - horizontalPassTexelHeightOffset = 0.0; + self->horizontalPassTexelWidthOffset = 1.0 / self->_originalImageSize.width; + self->horizontalPassTexelHeightOffset = 0.0; }); } diff --git a/ScannerApp/Frameworks/GPUImage/framework/Source/GPUImageLineGenerator.m b/ScannerApp/Frameworks/GPUImage/framework/Source/GPUImageLineGenerator.m index 85d93be..52b8662 100644 --- a/ScannerApp/Frameworks/GPUImage/framework/Source/GPUImageLineGenerator.m +++ b/ScannerApp/Frameworks/GPUImage/framework/Source/GPUImageLineGenerator.m @@ -53,8 +53,8 @@ - (id)init; } runSynchronouslyOnVideoProcessingQueue(^{ - lineWidthUniform = [filterProgram uniformIndex:@"lineWidth"]; - lineColorUniform = [filterProgram uniformIndex:@"lineColor"]; + self->lineWidthUniform = [self->filterProgram uniformIndex:@"lineWidth"]; + self->lineColorUniform = [self->filterProgram uniformIndex:@"lineColor"]; self.lineWidth = 1.0; [self setLineColorRed:0.0 green:1.0 blue:0.0]; @@ -117,10 +117,10 @@ - (void)renderLinesFromArray:(GLfloat *)lineSlopeAndIntercepts count:(NSUInteger } runSynchronouslyOnVideoProcessingQueue(^{ - [GPUImageContext setActiveShaderProgram:filterProgram]; + [GPUImageContext setActiveShaderProgram:self->filterProgram]; - outputFramebuffer = [[GPUImageContext sharedFramebufferCache] fetchFramebufferForSize:[self sizeOfFBO] textureOptions:self.outputTextureOptions onlyTexture:NO]; - [outputFramebuffer activateFramebuffer]; + self->outputFramebuffer = [[GPUImageContext sharedFramebufferCache] fetchFramebufferForSize:[self sizeOfFBO] textureOptions:self.outputTextureOptions onlyTexture:NO]; + [self->outputFramebuffer activateFramebuffer]; glClearColor(0.0, 0.0, 0.0, 0.0); glClear(GL_COLOR_BUFFER_BIT); @@ -129,7 +129,7 @@ - (void)renderLinesFromArray:(GLfloat *)lineSlopeAndIntercepts count:(NSUInteger glBlendFunc(GL_ONE, GL_ONE); glEnable(GL_BLEND); - glVertexAttribPointer(filterPositionAttribute, 2, GL_FLOAT, 0, 0, lineCoordinates); + glVertexAttribPointer(self->filterPositionAttribute, 2, GL_FLOAT, 0, 0, self->lineCoordinates); glDrawArrays(GL_LINES, 0, ((unsigned int)numberOfLines * 2)); glDisable(GL_BLEND); diff --git a/ScannerApp/Frameworks/GPUImage/framework/Source/GPUImageLuminosity.m b/ScannerApp/Frameworks/GPUImage/framework/Source/GPUImageLuminosity.m index 37f374a..f3b3cab 100644 --- a/ScannerApp/Frameworks/GPUImage/framework/Source/GPUImageLuminosity.m +++ b/ScannerApp/Frameworks/GPUImage/framework/Source/GPUImageLuminosity.m @@ -127,37 +127,37 @@ - (id)init; runSynchronouslyOnVideoProcessingQueue(^{ [GPUImageContext useImageProcessingContext]; - secondFilterProgram = [[GPUImageContext sharedImageProcessingContext] programForVertexShaderString:kGPUImageColorAveragingVertexShaderString fragmentShaderString:kGPUImageLuminosityFragmentShaderString]; + self->secondFilterProgram = [[GPUImageContext sharedImageProcessingContext] programForVertexShaderString:kGPUImageColorAveragingVertexShaderString fragmentShaderString:kGPUImageLuminosityFragmentShaderString]; - if (!secondFilterProgram.initialized) + if (!self->secondFilterProgram.initialized) { [self initializeSecondaryAttributes]; - if (![secondFilterProgram link]) + if (![self->secondFilterProgram link]) { - NSString *progLog = [secondFilterProgram programLog]; + NSString *progLog = [self->secondFilterProgram programLog]; NSLog(@"Program link log: %@", progLog); - NSString *fragLog = [secondFilterProgram fragmentShaderLog]; + NSString *fragLog = [self->secondFilterProgram fragmentShaderLog]; NSLog(@"Fragment shader compile log: %@", fragLog); - NSString *vertLog = [secondFilterProgram vertexShaderLog]; + NSString *vertLog = [self->secondFilterProgram vertexShaderLog]; NSLog(@"Vertex shader compile log: %@", vertLog); - filterProgram = nil; + self->filterProgram = nil; NSAssert(NO, @"Filter shader link failed"); } } - secondFilterPositionAttribute = [secondFilterProgram attributeIndex:@"position"]; - secondFilterTextureCoordinateAttribute = [secondFilterProgram attributeIndex:@"inputTextureCoordinate"]; - secondFilterInputTextureUniform = [secondFilterProgram uniformIndex:@"inputImageTexture"]; // This does assume a name of "inputImageTexture" for the fragment shader - secondFilterInputTextureUniform2 = [secondFilterProgram uniformIndex:@"inputImageTexture2"]; // This does assume a name of "inputImageTexture2" for second input texture in the fragment shader + self->secondFilterPositionAttribute = [self->secondFilterProgram attributeIndex:@"position"]; + self->secondFilterTextureCoordinateAttribute = [self->secondFilterProgram attributeIndex:@"inputTextureCoordinate"]; + self->secondFilterInputTextureUniform = [self->secondFilterProgram uniformIndex:@"inputImageTexture"]; // This does assume a name of "inputImageTexture" for the fragment shader + self->secondFilterInputTextureUniform2 = [self->secondFilterProgram uniformIndex:@"inputImageTexture2"]; // This does assume a name of "inputImageTexture2" for second input texture in the fragment shader - secondFilterTexelWidthUniform = [secondFilterProgram uniformIndex:@"texelWidth"]; - secondFilterTexelHeightUniform = [secondFilterProgram uniformIndex:@"texelHeight"]; + self->secondFilterTexelWidthUniform = [self->secondFilterProgram uniformIndex:@"texelWidth"]; + self->secondFilterTexelHeightUniform = [self->secondFilterProgram uniformIndex:@"texelHeight"]; - [GPUImageContext setActiveShaderProgram:secondFilterProgram]; + [GPUImageContext setActiveShaderProgram:self->secondFilterProgram]; - glEnableVertexAttribArray(secondFilterPositionAttribute); - glEnableVertexAttribArray(secondFilterTextureCoordinateAttribute); + glEnableVertexAttribArray(self->secondFilterPositionAttribute); + glEnableVertexAttribArray(self->secondFilterTextureCoordinateAttribute); }); return self; @@ -296,31 +296,31 @@ - (void)extractLuminosityAtFrameTime:(CMTime)frameTime; NSAssert(self.outputTextureOptions.internalFormat == GL_RGBA, @"The output texture format for this filter must be GL_RGBA."); NSAssert(self.outputTextureOptions.type == GL_UNSIGNED_BYTE, @"The type of the output texture of this filter must be GL_UNSIGNED_BYTE."); - NSUInteger totalNumberOfPixels = round(finalStageSize.width * finalStageSize.height); + NSUInteger totalNumberOfPixels = round(self->finalStageSize.width * self->finalStageSize.height); - if (rawImagePixels == NULL) + if (self->rawImagePixels == NULL) { - rawImagePixels = (GLubyte *)malloc(totalNumberOfPixels * 4); + self->rawImagePixels = (GLubyte *)malloc(totalNumberOfPixels * 4); } [GPUImageContext useImageProcessingContext]; - [outputFramebuffer activateFramebuffer]; + [self->outputFramebuffer activateFramebuffer]; - glReadPixels(0, 0, (int)finalStageSize.width, (int)finalStageSize.height, GL_RGBA, GL_UNSIGNED_BYTE, rawImagePixels); + glReadPixels(0, 0, (int)self->finalStageSize.width, (int)self->finalStageSize.height, GL_RGBA, GL_UNSIGNED_BYTE, self->rawImagePixels); NSUInteger luminanceTotal = 0; NSUInteger byteIndex = 0; for (NSUInteger currentPixel = 0; currentPixel < totalNumberOfPixels; currentPixel++) { - luminanceTotal += rawImagePixels[byteIndex]; + luminanceTotal += self->rawImagePixels[byteIndex]; byteIndex += 4; } CGFloat normalizedLuminosityTotal = (CGFloat)luminanceTotal / (CGFloat)totalNumberOfPixels / 255.0; - if (_luminosityProcessingFinishedBlock != NULL) + if (self->_luminosityProcessingFinishedBlock != NULL) { - _luminosityProcessingFinishedBlock(normalizedLuminosityTotal, frameTime); + self->_luminosityProcessingFinishedBlock(normalizedLuminosityTotal, frameTime); } }); } diff --git a/ScannerApp/Frameworks/GPUImage/framework/Source/GPUImageMovie.m b/ScannerApp/Frameworks/GPUImage/framework/Source/GPUImageMovie.m index 4da050f..7e3de6e 100755 --- a/ScannerApp/Frameworks/GPUImage/framework/Source/GPUImageMovie.m +++ b/ScannerApp/Frameworks/GPUImage/framework/Source/GPUImageMovie.m @@ -101,38 +101,38 @@ - (void)yuvConversionSetup; runSynchronouslyOnVideoProcessingQueue(^{ [GPUImageContext useImageProcessingContext]; - _preferredConversion = kColorConversion709; - isFullYUVRange = YES; - yuvConversionProgram = [[GPUImageContext sharedImageProcessingContext] programForVertexShaderString:kGPUImageVertexShaderString fragmentShaderString:kGPUImageYUVFullRangeConversionForLAFragmentShaderString]; + self->_preferredConversion = kColorConversion709; + self->isFullYUVRange = YES; + self->yuvConversionProgram = [[GPUImageContext sharedImageProcessingContext] programForVertexShaderString:kGPUImageVertexShaderString fragmentShaderString:kGPUImageYUVFullRangeConversionForLAFragmentShaderString]; - if (!yuvConversionProgram.initialized) + if (!self->yuvConversionProgram.initialized) { - [yuvConversionProgram addAttribute:@"position"]; - [yuvConversionProgram addAttribute:@"inputTextureCoordinate"]; + [self->yuvConversionProgram addAttribute:@"position"]; + [self->yuvConversionProgram addAttribute:@"inputTextureCoordinate"]; - if (![yuvConversionProgram link]) + if (![self->yuvConversionProgram link]) { - NSString *progLog = [yuvConversionProgram programLog]; + NSString *progLog = [self->yuvConversionProgram programLog]; NSLog(@"Program link log: %@", progLog); - NSString *fragLog = [yuvConversionProgram fragmentShaderLog]; + NSString *fragLog = [self->yuvConversionProgram fragmentShaderLog]; NSLog(@"Fragment shader compile log: %@", fragLog); - NSString *vertLog = [yuvConversionProgram vertexShaderLog]; + NSString *vertLog = [self->yuvConversionProgram vertexShaderLog]; NSLog(@"Vertex shader compile log: %@", vertLog); - yuvConversionProgram = nil; + self->yuvConversionProgram = nil; NSAssert(NO, @"Filter shader link failed"); } } - yuvConversionPositionAttribute = [yuvConversionProgram attributeIndex:@"position"]; - yuvConversionTextureCoordinateAttribute = [yuvConversionProgram attributeIndex:@"inputTextureCoordinate"]; - yuvConversionLuminanceTextureUniform = [yuvConversionProgram uniformIndex:@"luminanceTexture"]; - yuvConversionChrominanceTextureUniform = [yuvConversionProgram uniformIndex:@"chrominanceTexture"]; - yuvConversionMatrixUniform = [yuvConversionProgram uniformIndex:@"colorConversionMatrix"]; + self->yuvConversionPositionAttribute = [self->yuvConversionProgram attributeIndex:@"position"]; + self->yuvConversionTextureCoordinateAttribute = [self->yuvConversionProgram attributeIndex:@"inputTextureCoordinate"]; + self->yuvConversionLuminanceTextureUniform = [self->yuvConversionProgram uniformIndex:@"luminanceTexture"]; + self->yuvConversionChrominanceTextureUniform = [self->yuvConversionProgram uniformIndex:@"chrominanceTexture"]; + self->yuvConversionMatrixUniform = [self->yuvConversionProgram uniformIndex:@"colorConversionMatrix"]; - [GPUImageContext setActiveShaderProgram:yuvConversionProgram]; + [GPUImageContext setActiveShaderProgram:self->yuvConversionProgram]; - glEnableVertexAttribArray(yuvConversionPositionAttribute); - glEnableVertexAttribArray(yuvConversionTextureCoordinateAttribute); + glEnableVertexAttribArray(self->yuvConversionPositionAttribute); + glEnableVertexAttribArray(self->yuvConversionTextureCoordinateAttribute); }); } } @@ -317,9 +317,9 @@ - (void)processPlayerItem runSynchronouslyOnVideoProcessingQueue(^{ #if TARGET_IPHONE_SIMULATOR || TARGET_OS_IPHONE - displayLink = [CADisplayLink displayLinkWithTarget:self selector:@selector(displayLinkCallback:)]; - [displayLink addToRunLoop:[NSRunLoop currentRunLoop] forMode:NSRunLoopCommonModes]; - [displayLink setPaused:YES]; + self->displayLink = [CADisplayLink displayLinkWithTarget:self selector:@selector(displayLinkCallback:)]; + [self->displayLink addToRunLoop:[NSRunLoop currentRunLoop] forMode:NSRunLoopCommonModes]; + [self->displayLink setPaused:YES]; #else // Suggested implementation: use CVDisplayLink http://stackoverflow.com/questions/14158743/alternative-of-cadisplaylink-for-mac-os-x CGDirectDisplayID displayID = CGMainDisplayID(); @@ -342,11 +342,11 @@ - (void)processPlayerItem else { [pixBuffAttributes setObject:@(kCVPixelFormatType_32BGRA) forKey:(id)kCVPixelBufferPixelFormatTypeKey]; } - playerItemOutput = [[AVPlayerItemVideoOutput alloc] initWithPixelBufferAttributes:pixBuffAttributes]; - [playerItemOutput setDelegate:self queue:videoProcessingQueue]; + self->playerItemOutput = [[AVPlayerItemVideoOutput alloc] initWithPixelBufferAttributes:pixBuffAttributes]; + [self->playerItemOutput setDelegate:self queue:videoProcessingQueue]; - [_playerItem addOutput:playerItemOutput]; - [playerItemOutput requestNotificationOfMediaDataChangeWithAdvanceInterval:0.1]; + [self->_playerItem addOutput:self->playerItemOutput]; + [self->playerItemOutput requestNotificationOfMediaDataChangeWithAdvanceInterval:0.1]; }); } diff --git a/ScannerApp/Frameworks/GPUImage/framework/Source/GPUImageOutput.m b/ScannerApp/Frameworks/GPUImage/framework/Source/GPUImageOutput.m index d9bdaef..56f8159 100755 --- a/ScannerApp/Frameworks/GPUImage/framework/Source/GPUImageOutput.m +++ b/ScannerApp/Frameworks/GPUImage/framework/Source/GPUImageOutput.m @@ -225,10 +225,10 @@ - (void)addTarget:(id)newTarget atTextureLocation:(NSInteger)text cachedMaximumOutputSize = CGSizeZero; runSynchronouslyOnVideoProcessingQueue(^{ [self setInputFramebufferForTarget:newTarget atIndex:textureLocation]; - [targets addObject:newTarget]; - [targetTextureIndices addObject:[NSNumber numberWithInteger:textureLocation]]; + [self->targets addObject:newTarget]; + [self->targetTextureIndices addObject:[NSNumber numberWithInteger:textureLocation]]; - allTargetsWantMonochromeData = allTargetsWantMonochromeData && [newTarget wantsMonochromeInput]; + self->allTargetsWantMonochromeData = self->allTargetsWantMonochromeData && [newTarget wantsMonochromeInput]; }); } @@ -253,8 +253,8 @@ - (void)removeTarget:(id)targetToRemove; [targetToRemove setInputSize:CGSizeZero atIndex:textureIndexOfTarget]; [targetToRemove setInputRotation:kGPUImageNoRotation atIndex:textureIndexOfTarget]; - [targetTextureIndices removeObjectAtIndex:indexOfObject]; - [targets removeObject:targetToRemove]; + [self->targetTextureIndices removeObjectAtIndex:indexOfObject]; + [self->targets removeObject:targetToRemove]; [targetToRemove endProcessing]; }); } @@ -263,18 +263,18 @@ - (void)removeAllTargets; { cachedMaximumOutputSize = CGSizeZero; runSynchronouslyOnVideoProcessingQueue(^{ - for (id targetToRemove in targets) + for (id targetToRemove in self->targets) { - NSInteger indexOfObject = [targets indexOfObject:targetToRemove]; - NSInteger textureIndexOfTarget = [[targetTextureIndices objectAtIndex:indexOfObject] integerValue]; + NSInteger indexOfObject = [self->targets indexOfObject:targetToRemove]; + NSInteger textureIndexOfTarget = [[self->targetTextureIndices objectAtIndex:indexOfObject] integerValue]; [targetToRemove setInputSize:CGSizeZero atIndex:textureIndexOfTarget]; [targetToRemove setInputRotation:kGPUImageNoRotation atIndex:textureIndexOfTarget]; } - [targets removeAllObjects]; - [targetTextureIndices removeAllObjects]; + [self->targets removeAllObjects]; + [self->targetTextureIndices removeAllObjects]; - allTargetsWantMonochromeData = YES; + self->allTargetsWantMonochromeData = YES; }); } diff --git a/ScannerApp/Frameworks/GPUImage/framework/Source/GPUImageRawDataInput.m b/ScannerApp/Frameworks/GPUImage/framework/Source/GPUImageRawDataInput.m index cfa3b12..05c2c87 100644 --- a/ScannerApp/Frameworks/GPUImage/framework/Source/GPUImageRawDataInput.m +++ b/ScannerApp/Frameworks/GPUImage/framework/Source/GPUImageRawDataInput.m @@ -93,17 +93,17 @@ - (void)processData; CGSize pixelSizeOfImage = [self outputImageSize]; - for (id currentTarget in targets) + for (id currentTarget in self->targets) { - NSInteger indexOfObject = [targets indexOfObject:currentTarget]; - NSInteger textureIndexOfTarget = [[targetTextureIndices objectAtIndex:indexOfObject] integerValue]; + NSInteger indexOfObject = [self->targets indexOfObject:currentTarget]; + NSInteger textureIndexOfTarget = [[self->targetTextureIndices objectAtIndex:indexOfObject] integerValue]; [currentTarget setInputSize:pixelSizeOfImage atIndex:textureIndexOfTarget]; - [currentTarget setInputFramebuffer:outputFramebuffer atIndex:textureIndexOfTarget]; + [currentTarget setInputFramebuffer:self->outputFramebuffer atIndex:textureIndexOfTarget]; [currentTarget newFrameReadyAtTime:kCMTimeInvalid atIndex:textureIndexOfTarget]; } - dispatch_semaphore_signal(dataUpdateSemaphore); + dispatch_semaphore_signal(self->dataUpdateSemaphore); }); } @@ -118,16 +118,16 @@ - (void)processDataForTimestamp:(CMTime)frameTime; CGSize pixelSizeOfImage = [self outputImageSize]; - for (id currentTarget in targets) + for (id currentTarget in self->targets) { - NSInteger indexOfObject = [targets indexOfObject:currentTarget]; - NSInteger textureIndexOfTarget = [[targetTextureIndices objectAtIndex:indexOfObject] integerValue]; + NSInteger indexOfObject = [self->targets indexOfObject:currentTarget]; + NSInteger textureIndexOfTarget = [[self->targetTextureIndices objectAtIndex:indexOfObject] integerValue]; [currentTarget setInputSize:pixelSizeOfImage atIndex:textureIndexOfTarget]; [currentTarget newFrameReadyAtTime:frameTime atIndex:textureIndexOfTarget]; } - dispatch_semaphore_signal(dataUpdateSemaphore); + dispatch_semaphore_signal(self->dataUpdateSemaphore); }); } diff --git a/ScannerApp/Frameworks/GPUImage/framework/Source/GPUImageRawDataOutput.m b/ScannerApp/Frameworks/GPUImage/framework/Source/GPUImageRawDataOutput.m index 18101e2..7f18c50 100755 --- a/ScannerApp/Frameworks/GPUImage/framework/Source/GPUImageRawDataOutput.m +++ b/ScannerApp/Frameworks/GPUImage/framework/Source/GPUImageRawDataOutput.m @@ -261,16 +261,16 @@ - (GLubyte *)rawBytesForImage; if ([GPUImageContext supportsFastTextureUpload]) { glFinish(); - _rawBytesForImage = [outputFramebuffer byteBuffer]; + self->_rawBytesForImage = [self->outputFramebuffer byteBuffer]; } else { - glReadPixels(0, 0, imageSize.width, imageSize.height, GL_RGBA, GL_UNSIGNED_BYTE, _rawBytesForImage); + glReadPixels(0, 0, self->imageSize.width, self->imageSize.height, GL_RGBA, GL_UNSIGNED_BYTE, self->_rawBytesForImage); // GL_EXT_read_format_bgra // glReadPixels(0, 0, imageSize.width, imageSize.height, GL_BGRA_EXT, GL_UNSIGNED_BYTE, _rawBytesForImage); } - hasReadFromTheCurrentFrame = YES; + self->hasReadFromTheCurrentFrame = YES; }); diff --git a/ScannerApp/Frameworks/GPUImage/framework/Source/GPUImageSharpenFilter.m b/ScannerApp/Frameworks/GPUImage/framework/Source/GPUImageSharpenFilter.m index 6d7367a..c14b037 100755 --- a/ScannerApp/Frameworks/GPUImage/framework/Source/GPUImageSharpenFilter.m +++ b/ScannerApp/Frameworks/GPUImage/framework/Source/GPUImageSharpenFilter.m @@ -118,17 +118,17 @@ - (id)init; - (void)setupFilterForSize:(CGSize)filterFrameSize; { runSynchronouslyOnVideoProcessingQueue(^{ - [GPUImageContext setActiveShaderProgram:filterProgram]; + [GPUImageContext setActiveShaderProgram:self->filterProgram]; - if (GPUImageRotationSwapsWidthAndHeight(inputRotation)) + if (GPUImageRotationSwapsWidthAndHeight(self->inputRotation)) { - glUniform1f(imageWidthFactorUniform, 1.0 / filterFrameSize.height); - glUniform1f(imageHeightFactorUniform, 1.0 / filterFrameSize.width); + glUniform1f(self->imageWidthFactorUniform, 1.0 / filterFrameSize.height); + glUniform1f(self->imageHeightFactorUniform, 1.0 / filterFrameSize.width); } else { - glUniform1f(imageWidthFactorUniform, 1.0 / filterFrameSize.width); - glUniform1f(imageHeightFactorUniform, 1.0 / filterFrameSize.height); + glUniform1f(self->imageWidthFactorUniform, 1.0 / filterFrameSize.width); + glUniform1f(self->imageHeightFactorUniform, 1.0 / filterFrameSize.height); } }); } diff --git a/ScannerApp/Frameworks/GPUImage/framework/Source/GPUImageSobelEdgeDetectionFilter.m b/ScannerApp/Frameworks/GPUImage/framework/Source/GPUImageSobelEdgeDetectionFilter.m index e193f02..7aac23f 100755 --- a/ScannerApp/Frameworks/GPUImage/framework/Source/GPUImageSobelEdgeDetectionFilter.m +++ b/ScannerApp/Frameworks/GPUImage/framework/Source/GPUImageSobelEdgeDetectionFilter.m @@ -126,9 +126,9 @@ - (void)setupFilterForSize:(CGSize)filterFrameSize; runSynchronouslyOnVideoProcessingQueue(^{ GLProgram *previousProgram = [GPUImageContext sharedImageProcessingContext].currentShaderProgram; - [GPUImageContext setActiveShaderProgram:secondFilterProgram]; - glUniform1f(texelWidthUniform, _texelWidth); - glUniform1f(texelHeightUniform, _texelHeight); + [GPUImageContext setActiveShaderProgram:self->secondFilterProgram]; + glUniform1f(self->texelWidthUniform, self->_texelWidth); + glUniform1f(self->texelHeightUniform, self->_texelHeight); [GPUImageContext setActiveShaderProgram:previousProgram]; }); } diff --git a/ScannerApp/Frameworks/GPUImage/framework/Source/GPUImageSolidColorGenerator.m b/ScannerApp/Frameworks/GPUImage/framework/Source/GPUImageSolidColorGenerator.m index 9b555ce..512910b 100644 --- a/ScannerApp/Frameworks/GPUImage/framework/Source/GPUImageSolidColorGenerator.m +++ b/ScannerApp/Frameworks/GPUImage/framework/Source/GPUImageSolidColorGenerator.m @@ -60,12 +60,12 @@ - (void)renderToTextureWithVertices:(const GLfloat *)vertices textureCoordinates } runSynchronouslyOnVideoProcessingQueue(^{ - [GPUImageContext setActiveShaderProgram:filterProgram]; + [GPUImageContext setActiveShaderProgram:self->filterProgram]; - outputFramebuffer = [[GPUImageContext sharedFramebufferCache] fetchFramebufferForSize:[self sizeOfFBO] textureOptions:self.outputTextureOptions onlyTexture:NO]; - [outputFramebuffer activateFramebuffer]; + self->outputFramebuffer = [[GPUImageContext sharedFramebufferCache] fetchFramebufferForSize:[self sizeOfFBO] textureOptions:self.outputTextureOptions onlyTexture:NO]; + [self->outputFramebuffer activateFramebuffer]; - glClearColor(_color.one, _color.two, _color.three, _color.four); + glClearColor(self->_color.one, self->_color.two, self->_color.three, self->_color.four); glClear(GL_COLOR_BUFFER_BIT); }); } diff --git a/ScannerApp/Frameworks/GPUImage/framework/Source/GPUImageStillCamera.m b/ScannerApp/Frameworks/GPUImage/framework/Source/GPUImageStillCamera.m index 447f79f..53737ab 100755 --- a/ScannerApp/Frameworks/GPUImage/framework/Source/GPUImageStillCamera.m +++ b/ScannerApp/Frameworks/GPUImage/framework/Source/GPUImageStillCamera.m @@ -161,7 +161,7 @@ - (void)capturePhotoAsImageProcessedUpToFilter:(GPUImageOutput *) if(!error){ filteredPhoto = [finalFilterInChain imageFromCurrentFramebuffer]; } - dispatch_semaphore_signal(frameRenderingSemaphore); + dispatch_semaphore_signal(self->frameRenderingSemaphore); block(filteredPhoto, error); }]; @@ -174,7 +174,7 @@ - (void)capturePhotoAsImageProcessedUpToFilter:(GPUImageOutput *) if(!error) { filteredPhoto = [finalFilterInChain imageFromCurrentFramebufferWithOrientation:orientation]; } - dispatch_semaphore_signal(frameRenderingSemaphore); + dispatch_semaphore_signal(self->frameRenderingSemaphore); block(filteredPhoto, error); }]; @@ -190,7 +190,7 @@ - (void)capturePhotoAsJPEGProcessedUpToFilter:(GPUImageOutput *)f if(!error){ @autoreleasepool { UIImage *filteredPhoto = [finalFilterInChain imageFromCurrentFramebuffer]; - dispatch_semaphore_signal(frameRenderingSemaphore); + dispatch_semaphore_signal(self->frameRenderingSemaphore); // reportAvailableMemoryForGPUImage(@"After UIImage generation"); dataForJPEGFile = UIImageJPEGRepresentation(filteredPhoto,self.jpegCompressionQuality); @@ -199,7 +199,7 @@ - (void)capturePhotoAsJPEGProcessedUpToFilter:(GPUImageOutput *)f // reportAvailableMemoryForGPUImage(@"After autorelease pool"); }else{ - dispatch_semaphore_signal(frameRenderingSemaphore); + dispatch_semaphore_signal(self->frameRenderingSemaphore); } block(dataForJPEGFile, error); @@ -213,12 +213,12 @@ - (void)capturePhotoAsJPEGProcessedUpToFilter:(GPUImageOutput *)f if(!error) { @autoreleasepool { UIImage *filteredPhoto = [finalFilterInChain imageFromCurrentFramebufferWithOrientation:orientation]; - dispatch_semaphore_signal(frameRenderingSemaphore); + dispatch_semaphore_signal(self->frameRenderingSemaphore); dataForJPEGFile = UIImageJPEGRepresentation(filteredPhoto, self.jpegCompressionQuality); } } else { - dispatch_semaphore_signal(frameRenderingSemaphore); + dispatch_semaphore_signal(self->frameRenderingSemaphore); } block(dataForJPEGFile, error); @@ -234,11 +234,11 @@ - (void)capturePhotoAsPNGProcessedUpToFilter:(GPUImageOutput *)fi if(!error){ @autoreleasepool { UIImage *filteredPhoto = [finalFilterInChain imageFromCurrentFramebuffer]; - dispatch_semaphore_signal(frameRenderingSemaphore); + dispatch_semaphore_signal(self->frameRenderingSemaphore); dataForPNGFile = UIImagePNGRepresentation(filteredPhoto); } }else{ - dispatch_semaphore_signal(frameRenderingSemaphore); + dispatch_semaphore_signal(self->frameRenderingSemaphore); } block(dataForPNGFile, error); @@ -256,11 +256,11 @@ - (void)capturePhotoAsPNGProcessedUpToFilter:(GPUImageOutput *)fi if(!error){ @autoreleasepool { UIImage *filteredPhoto = [finalFilterInChain imageFromCurrentFramebufferWithOrientation:orientation]; - dispatch_semaphore_signal(frameRenderingSemaphore); + dispatch_semaphore_signal(self->frameRenderingSemaphore); dataForPNGFile = UIImagePNGRepresentation(filteredPhoto); } }else{ - dispatch_semaphore_signal(frameRenderingSemaphore); + dispatch_semaphore_signal(self->frameRenderingSemaphore); } block(dataForPNGFile, error); @@ -304,32 +304,32 @@ - (void)capturePhotoProcessedUpToFilter:(GPUImageOutput *)finalFi GPUImageCreateResizedSampleBuffer(cameraFrame, scaledImageSizeToFitOnGPU, &sampleBuffer); } - dispatch_semaphore_signal(frameRenderingSemaphore); + dispatch_semaphore_signal(self->frameRenderingSemaphore); [finalFilterInChain useNextFrameForImageCapture]; - [self captureOutput:photoOutput didOutputSampleBuffer:sampleBuffer fromConnection:[[photoOutput connections] objectAtIndex:0]]; - dispatch_semaphore_wait(frameRenderingSemaphore, DISPATCH_TIME_FOREVER); + [self captureOutput:self->photoOutput didOutputSampleBuffer:sampleBuffer fromConnection:[[self->photoOutput connections] objectAtIndex:0]]; + dispatch_semaphore_wait(self->frameRenderingSemaphore, DISPATCH_TIME_FOREVER); if (sampleBuffer != NULL) CFRelease(sampleBuffer); } else { // This is a workaround for the corrupt images that are sometimes returned when taking a photo with the front camera and using the iOS 5.0 texture caches - AVCaptureDevicePosition currentCameraPosition = [[videoInput device] position]; - if ( (currentCameraPosition != AVCaptureDevicePositionFront) || (![GPUImageContext supportsFastTextureUpload]) || !requiresFrontCameraTextureCacheCorruptionWorkaround) + AVCaptureDevicePosition currentCameraPosition = [[self->videoInput device] position]; + if ( (currentCameraPosition != AVCaptureDevicePositionFront) || (![GPUImageContext supportsFastTextureUpload]) || !self->requiresFrontCameraTextureCacheCorruptionWorkaround) { - dispatch_semaphore_signal(frameRenderingSemaphore); + dispatch_semaphore_signal(self->frameRenderingSemaphore); [finalFilterInChain useNextFrameForImageCapture]; - [self captureOutput:photoOutput didOutputSampleBuffer:imageSampleBuffer fromConnection:[[photoOutput connections] objectAtIndex:0]]; - dispatch_semaphore_wait(frameRenderingSemaphore, DISPATCH_TIME_FOREVER); + [self captureOutput:self->photoOutput didOutputSampleBuffer:imageSampleBuffer fromConnection:[[self->photoOutput connections] objectAtIndex:0]]; + dispatch_semaphore_wait(self->frameRenderingSemaphore, DISPATCH_TIME_FOREVER); } } CFDictionaryRef metadata = CMCopyDictionaryOfAttachments(NULL, imageSampleBuffer, kCMAttachmentMode_ShouldPropagate); - _currentCaptureMetadata = (__bridge_transfer NSDictionary *)metadata; + self->_currentCaptureMetadata = (__bridge_transfer NSDictionary *)metadata; block(nil); - _currentCaptureMetadata = nil; + self->_currentCaptureMetadata = nil; }]; } diff --git a/ScannerApp/Frameworks/GPUImage/framework/Source/GPUImageTextureInput.m b/ScannerApp/Frameworks/GPUImage/framework/Source/GPUImageTextureInput.m index ad3ca1d..c7f62cc 100755 --- a/ScannerApp/Frameworks/GPUImage/framework/Source/GPUImageTextureInput.m +++ b/ScannerApp/Frameworks/GPUImage/framework/Source/GPUImageTextureInput.m @@ -19,7 +19,7 @@ - (id)initWithTexture:(GLuint)newInputTexture size:(CGSize)newTextureSize; textureSize = newTextureSize; runSynchronouslyOnVideoProcessingQueue(^{ - outputFramebuffer = [[GPUImageFramebuffer alloc] initWithSize:newTextureSize overriddenTexture:newInputTexture]; + self->outputFramebuffer = [[GPUImageFramebuffer alloc] initWithSize:newTextureSize overriddenTexture:newInputTexture]; }); return self; @@ -31,13 +31,13 @@ - (id)initWithTexture:(GLuint)newInputTexture size:(CGSize)newTextureSize; - (void)processTextureWithFrameTime:(CMTime)frameTime; { runAsynchronouslyOnVideoProcessingQueue(^{ - for (id currentTarget in targets) + for (id currentTarget in self->targets) { - NSInteger indexOfObject = [targets indexOfObject:currentTarget]; - NSInteger targetTextureIndex = [[targetTextureIndices objectAtIndex:indexOfObject] integerValue]; + NSInteger indexOfObject = [self->targets indexOfObject:currentTarget]; + NSInteger targetTextureIndex = [[self->targetTextureIndices objectAtIndex:indexOfObject] integerValue]; - [currentTarget setInputSize:textureSize atIndex:targetTextureIndex]; - [currentTarget setInputFramebuffer:outputFramebuffer atIndex:targetTextureIndex]; + [currentTarget setInputSize:self->textureSize atIndex:targetTextureIndex]; + [currentTarget setInputFramebuffer:self->outputFramebuffer atIndex:targetTextureIndex]; [currentTarget newFrameReadyAtTime:frameTime atIndex:targetTextureIndex]; } }); diff --git a/ScannerApp/Frameworks/GPUImage/framework/Source/GPUImageThreeInputFilter.m b/ScannerApp/Frameworks/GPUImage/framework/Source/GPUImageThreeInputFilter.m index 2f4f113..89f62da 100644 --- a/ScannerApp/Frameworks/GPUImage/framework/Source/GPUImageThreeInputFilter.m +++ b/ScannerApp/Frameworks/GPUImage/framework/Source/GPUImageThreeInputFilter.m @@ -55,10 +55,10 @@ - (id)initWithVertexShaderFromString:(NSString *)vertexShaderString fragmentShad runSynchronouslyOnVideoProcessingQueue(^{ [GPUImageContext useImageProcessingContext]; - filterThirdTextureCoordinateAttribute = [filterProgram attributeIndex:@"inputTextureCoordinate3"]; + self->filterThirdTextureCoordinateAttribute = [self->filterProgram attributeIndex:@"inputTextureCoordinate3"]; - filterInputTextureUniform3 = [filterProgram uniformIndex:@"inputImageTexture3"]; // This does assume a name of "inputImageTexture3" for the third input texture in the fragment shader - glEnableVertexAttribArray(filterThirdTextureCoordinateAttribute); + self->filterInputTextureUniform3 = [self->filterProgram uniformIndex:@"inputImageTexture3"]; // This does assume a name of "inputImageTexture3" for the third input texture in the fragment shader + glEnableVertexAttribArray(self->filterThirdTextureCoordinateAttribute); }); return self; diff --git a/ScannerApp/Frameworks/GPUImage/framework/Source/GPUImageToneCurveFilter.m b/ScannerApp/Frameworks/GPUImage/framework/Source/GPUImageToneCurveFilter.m index 18a717e..fa2f9d3 100644 --- a/ScannerApp/Frameworks/GPUImage/framework/Source/GPUImageToneCurveFilter.m +++ b/ScannerApp/Frameworks/GPUImage/framework/Source/GPUImageToneCurveFilter.m @@ -232,11 +232,11 @@ - (void)dealloc runSynchronouslyOnVideoProcessingQueue(^{ [GPUImageContext useImageProcessingContext]; - if (toneCurveTexture) + if (self->toneCurveTexture) { - glDeleteTextures(1, &toneCurveTexture); - toneCurveTexture = 0; - free(toneCurveByteArray); + glDeleteTextures(1, &self->toneCurveTexture); + self->toneCurveTexture = 0; + free(self->toneCurveByteArray); } }); } @@ -487,39 +487,39 @@ - (void)updateToneCurveTexture; { runSynchronouslyOnVideoProcessingQueue(^{ [GPUImageContext useImageProcessingContext]; - if (!toneCurveTexture) + if (!self->toneCurveTexture) { glActiveTexture(GL_TEXTURE3); - glGenTextures(1, &toneCurveTexture); - glBindTexture(GL_TEXTURE_2D, toneCurveTexture); + glGenTextures(1, &self->toneCurveTexture); + glBindTexture(GL_TEXTURE_2D, self->toneCurveTexture); glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_LINEAR); glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_LINEAR); glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_EDGE); glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE); - toneCurveByteArray = calloc(256 * 4, sizeof(GLubyte)); + self->toneCurveByteArray = calloc(256 * 4, sizeof(GLubyte)); } else { glActiveTexture(GL_TEXTURE3); - glBindTexture(GL_TEXTURE_2D, toneCurveTexture); + glBindTexture(GL_TEXTURE_2D, self->toneCurveTexture); } - if ( ([_redCurve count] >= 256) && ([_greenCurve count] >= 256) && ([_blueCurve count] >= 256) && ([_rgbCompositeCurve count] >= 256)) + if ( ([self->_redCurve count] >= 256) && ([self->_greenCurve count] >= 256) && ([self->_blueCurve count] >= 256) && ([self->_rgbCompositeCurve count] >= 256)) { for (unsigned int currentCurveIndex = 0; currentCurveIndex < 256; currentCurveIndex++) { // BGRA for upload to texture - GLubyte b = fmin(fmax(currentCurveIndex + [[_blueCurve objectAtIndex:currentCurveIndex] floatValue], 0), 255); - toneCurveByteArray[currentCurveIndex * 4] = fmin(fmax(b + [[_rgbCompositeCurve objectAtIndex:b] floatValue], 0), 255); - GLubyte g = fmin(fmax(currentCurveIndex + [[_greenCurve objectAtIndex:currentCurveIndex] floatValue], 0), 255); - toneCurveByteArray[currentCurveIndex * 4 + 1] = fmin(fmax(g + [[_rgbCompositeCurve objectAtIndex:g] floatValue], 0), 255); - GLubyte r = fmin(fmax(currentCurveIndex + [[_redCurve objectAtIndex:currentCurveIndex] floatValue], 0), 255); - toneCurveByteArray[currentCurveIndex * 4 + 2] = fmin(fmax(r + [[_rgbCompositeCurve objectAtIndex:r] floatValue], 0), 255); - toneCurveByteArray[currentCurveIndex * 4 + 3] = 255; + GLubyte b = fmin(fmax(currentCurveIndex + [[self->_blueCurve objectAtIndex:currentCurveIndex] floatValue], 0), 255); + self->toneCurveByteArray[currentCurveIndex * 4] = fmin(fmax(b + [[self->_rgbCompositeCurve objectAtIndex:b] floatValue], 0), 255); + GLubyte g = fmin(fmax(currentCurveIndex + [[self->_greenCurve objectAtIndex:currentCurveIndex] floatValue], 0), 255); + self->toneCurveByteArray[currentCurveIndex * 4 + 1] = fmin(fmax(g + [[self->_rgbCompositeCurve objectAtIndex:g] floatValue], 0), 255); + GLubyte r = fmin(fmax(currentCurveIndex + [[self->_redCurve objectAtIndex:currentCurveIndex] floatValue], 0), 255); + self->toneCurveByteArray[currentCurveIndex * 4 + 2] = fmin(fmax(r + [[self->_rgbCompositeCurve objectAtIndex:r] floatValue], 0), 255); + self->toneCurveByteArray[currentCurveIndex * 4 + 3] = 255; } - glTexImage2D(GL_TEXTURE_2D, 0, GL_RGBA, 256 /*width*/, 1 /*height*/, 0, GL_BGRA, GL_UNSIGNED_BYTE, toneCurveByteArray); + glTexImage2D(GL_TEXTURE_2D, 0, GL_RGBA, 256 /*width*/, 1 /*height*/, 0, GL_BGRA, GL_UNSIGNED_BYTE, self->toneCurveByteArray); } }); } diff --git a/ScannerApp/Frameworks/GPUImage/framework/Source/GPUImageTwoInputCrossTextureSamplingFilter.m b/ScannerApp/Frameworks/GPUImage/framework/Source/GPUImageTwoInputCrossTextureSamplingFilter.m index aa338f8..6d9b6b3 100644 --- a/ScannerApp/Frameworks/GPUImage/framework/Source/GPUImageTwoInputCrossTextureSamplingFilter.m +++ b/ScannerApp/Frameworks/GPUImage/framework/Source/GPUImageTwoInputCrossTextureSamplingFilter.m @@ -71,16 +71,16 @@ - (void)setupFilterForSize:(CGSize)filterFrameSize; _texelHeight = 1.0 / filterFrameSize.height; runSynchronouslyOnVideoProcessingQueue(^{ - [GPUImageContext setActiveShaderProgram:filterProgram]; - if (GPUImageRotationSwapsWidthAndHeight(inputRotation)) + [GPUImageContext setActiveShaderProgram:self->filterProgram]; + if (GPUImageRotationSwapsWidthAndHeight(self->inputRotation)) { - glUniform1f(texelWidthUniform, _texelHeight); - glUniform1f(texelHeightUniform, _texelWidth); + glUniform1f(self->texelWidthUniform, self->_texelHeight); + glUniform1f(self->texelHeightUniform, self->_texelWidth); } else { - glUniform1f(texelWidthUniform, _texelWidth); - glUniform1f(texelHeightUniform, _texelHeight); + glUniform1f(self->texelWidthUniform, self->_texelWidth); + glUniform1f(self->texelHeightUniform, self->_texelHeight); } }); } diff --git a/ScannerApp/Frameworks/GPUImage/framework/Source/GPUImageTwoInputFilter.m b/ScannerApp/Frameworks/GPUImage/framework/Source/GPUImageTwoInputFilter.m index cf31873..17c7a97 100644 --- a/ScannerApp/Frameworks/GPUImage/framework/Source/GPUImageTwoInputFilter.m +++ b/ScannerApp/Frameworks/GPUImage/framework/Source/GPUImageTwoInputFilter.m @@ -56,10 +56,10 @@ - (id)initWithVertexShaderFromString:(NSString *)vertexShaderString fragmentShad runSynchronouslyOnVideoProcessingQueue(^{ [GPUImageContext useImageProcessingContext]; - filterSecondTextureCoordinateAttribute = [filterProgram attributeIndex:@"inputTextureCoordinate2"]; + self->filterSecondTextureCoordinateAttribute = [self->filterProgram attributeIndex:@"inputTextureCoordinate2"]; - filterInputTextureUniform2 = [filterProgram uniformIndex:@"inputImageTexture2"]; // This does assume a name of "inputImageTexture2" for second input texture in the fragment shader - glEnableVertexAttribArray(filterSecondTextureCoordinateAttribute); + self->filterInputTextureUniform2 = [self->filterProgram uniformIndex:@"inputImageTexture2"]; // This does assume a name of "inputImageTexture2" for second input texture in the fragment shader + glEnableVertexAttribArray(self->filterSecondTextureCoordinateAttribute); }); return self; diff --git a/ScannerApp/Frameworks/GPUImage/framework/Source/GPUImageTwoPassFilter.m b/ScannerApp/Frameworks/GPUImage/framework/Source/GPUImageTwoPassFilter.m index 9eb292b..da20514 100755 --- a/ScannerApp/Frameworks/GPUImage/framework/Source/GPUImageTwoPassFilter.m +++ b/ScannerApp/Frameworks/GPUImage/framework/Source/GPUImageTwoPassFilter.m @@ -17,34 +17,34 @@ - (id)initWithFirstStageVertexShaderFromString:(NSString *)firstStageVertexShade runSynchronouslyOnVideoProcessingQueue(^{ [GPUImageContext useImageProcessingContext]; - secondFilterProgram = [[GPUImageContext sharedImageProcessingContext] programForVertexShaderString:secondStageVertexShaderString fragmentShaderString:secondStageFragmentShaderString]; + self->secondFilterProgram = [[GPUImageContext sharedImageProcessingContext] programForVertexShaderString:secondStageVertexShaderString fragmentShaderString:secondStageFragmentShaderString]; - if (!secondFilterProgram.initialized) + if (!self->secondFilterProgram.initialized) { [self initializeSecondaryAttributes]; - if (![secondFilterProgram link]) + if (![self->secondFilterProgram link]) { - NSString *progLog = [secondFilterProgram programLog]; + NSString *progLog = [self->secondFilterProgram programLog]; NSLog(@"Program link log: %@", progLog); - NSString *fragLog = [secondFilterProgram fragmentShaderLog]; + NSString *fragLog = [self->secondFilterProgram fragmentShaderLog]; NSLog(@"Fragment shader compile log: %@", fragLog); - NSString *vertLog = [secondFilterProgram vertexShaderLog]; + NSString *vertLog = [self->secondFilterProgram vertexShaderLog]; NSLog(@"Vertex shader compile log: %@", vertLog); - secondFilterProgram = nil; + self->secondFilterProgram = nil; NSAssert(NO, @"Filter shader link failed"); } } - secondFilterPositionAttribute = [secondFilterProgram attributeIndex:@"position"]; - secondFilterTextureCoordinateAttribute = [secondFilterProgram attributeIndex:@"inputTextureCoordinate"]; - secondFilterInputTextureUniform = [secondFilterProgram uniformIndex:@"inputImageTexture"]; // This does assume a name of "inputImageTexture" for the fragment shader - secondFilterInputTextureUniform2 = [secondFilterProgram uniformIndex:@"inputImageTexture2"]; // This does assume a name of "inputImageTexture2" for second input texture in the fragment shader + self->secondFilterPositionAttribute = [self->secondFilterProgram attributeIndex:@"position"]; + self->secondFilterTextureCoordinateAttribute = [self->secondFilterProgram attributeIndex:@"inputTextureCoordinate"]; + self->secondFilterInputTextureUniform = [self->secondFilterProgram uniformIndex:@"inputImageTexture"]; // This does assume a name of "inputImageTexture" for the fragment shader + self->secondFilterInputTextureUniform2 = [self->secondFilterProgram uniformIndex:@"inputImageTexture2"]; // This does assume a name of "inputImageTexture2" for second input texture in the fragment shader - [GPUImageContext setActiveShaderProgram:secondFilterProgram]; + [GPUImageContext setActiveShaderProgram:self->secondFilterProgram]; - glEnableVertexAttribArray(secondFilterPositionAttribute); - glEnableVertexAttribArray(secondFilterTextureCoordinateAttribute); + glEnableVertexAttribArray(self->secondFilterPositionAttribute); + glEnableVertexAttribArray(self->secondFilterTextureCoordinateAttribute); }); return self; diff --git a/ScannerApp/Frameworks/GPUImage/framework/Source/GPUImageTwoPassTextureSamplingFilter.m b/ScannerApp/Frameworks/GPUImage/framework/Source/GPUImageTwoPassTextureSamplingFilter.m index b6a2ec5..2511181 100644 --- a/ScannerApp/Frameworks/GPUImage/framework/Source/GPUImageTwoPassTextureSamplingFilter.m +++ b/ScannerApp/Frameworks/GPUImage/framework/Source/GPUImageTwoPassTextureSamplingFilter.m @@ -18,11 +18,11 @@ - (id)initWithFirstStageVertexShaderFromString:(NSString *)firstStageVertexShade runSynchronouslyOnVideoProcessingQueue(^{ [GPUImageContext useImageProcessingContext]; - verticalPassTexelWidthOffsetUniform = [filterProgram uniformIndex:@"texelWidthOffset"]; - verticalPassTexelHeightOffsetUniform = [filterProgram uniformIndex:@"texelHeightOffset"]; + self->verticalPassTexelWidthOffsetUniform = [self->filterProgram uniformIndex:@"texelWidthOffset"]; + self->verticalPassTexelHeightOffsetUniform = [self->filterProgram uniformIndex:@"texelHeightOffset"]; - horizontalPassTexelWidthOffsetUniform = [secondFilterProgram uniformIndex:@"texelWidthOffset"]; - horizontalPassTexelHeightOffsetUniform = [secondFilterProgram uniformIndex:@"texelHeightOffset"]; + self->horizontalPassTexelWidthOffsetUniform = [self->secondFilterProgram uniformIndex:@"texelWidthOffset"]; + self->horizontalPassTexelHeightOffsetUniform = [self->secondFilterProgram uniformIndex:@"texelHeightOffset"]; }); self.verticalTexelSpacing = 1.0; @@ -51,19 +51,19 @@ - (void)setupFilterForSize:(CGSize)filterFrameSize; { runSynchronouslyOnVideoProcessingQueue(^{ // The first pass through the framebuffer may rotate the inbound image, so need to account for that by changing up the kernel ordering for that pass - if (GPUImageRotationSwapsWidthAndHeight(inputRotation)) + if (GPUImageRotationSwapsWidthAndHeight(self->inputRotation)) { - verticalPassTexelWidthOffset = _verticalTexelSpacing / filterFrameSize.height; - verticalPassTexelHeightOffset = 0.0; + self->verticalPassTexelWidthOffset = self->_verticalTexelSpacing / filterFrameSize.height; + self->verticalPassTexelHeightOffset = 0.0; } else { - verticalPassTexelWidthOffset = 0.0; - verticalPassTexelHeightOffset = _verticalTexelSpacing / filterFrameSize.height; + self->verticalPassTexelWidthOffset = 0.0; + self->verticalPassTexelHeightOffset = self->_verticalTexelSpacing / filterFrameSize.height; } - horizontalPassTexelWidthOffset = _horizontalTexelSpacing / filterFrameSize.width; - horizontalPassTexelHeightOffset = 0.0; + self->horizontalPassTexelWidthOffset = self->_horizontalTexelSpacing / filterFrameSize.width; + self->horizontalPassTexelHeightOffset = 0.0; }); } diff --git a/ScannerApp/Frameworks/GPUImage/framework/Source/GPUImageVideoCamera.m b/ScannerApp/Frameworks/GPUImage/framework/Source/GPUImageVideoCamera.m index 18aa60c..9e9721e 100644 --- a/ScannerApp/Frameworks/GPUImage/framework/Source/GPUImageVideoCamera.m +++ b/ScannerApp/Frameworks/GPUImage/framework/Source/GPUImageVideoCamera.m @@ -153,7 +153,7 @@ - (id)initWithSessionPreset:(NSString *)sessionPreset cameraPosition:(AVCaptureD runSynchronouslyOnVideoProcessingQueue(^{ - if (captureAsYUV) + if (self->captureAsYUV) { [GPUImageContext useImageProcessingContext]; // if ([GPUImageContext deviceSupportsRedTextures]) @@ -162,45 +162,45 @@ - (id)initWithSessionPreset:(NSString *)sessionPreset cameraPosition:(AVCaptureD // } // else // { - if (isFullYUVRange) + if (self->isFullYUVRange) { - yuvConversionProgram = [[GPUImageContext sharedImageProcessingContext] programForVertexShaderString:kGPUImageVertexShaderString fragmentShaderString:kGPUImageYUVFullRangeConversionForLAFragmentShaderString]; + self->yuvConversionProgram = [[GPUImageContext sharedImageProcessingContext] programForVertexShaderString:kGPUImageVertexShaderString fragmentShaderString:kGPUImageYUVFullRangeConversionForLAFragmentShaderString]; } else { - yuvConversionProgram = [[GPUImageContext sharedImageProcessingContext] programForVertexShaderString:kGPUImageVertexShaderString fragmentShaderString:kGPUImageYUVVideoRangeConversionForLAFragmentShaderString]; + self->yuvConversionProgram = [[GPUImageContext sharedImageProcessingContext] programForVertexShaderString:kGPUImageVertexShaderString fragmentShaderString:kGPUImageYUVVideoRangeConversionForLAFragmentShaderString]; } // } - if (!yuvConversionProgram.initialized) + if (!self->yuvConversionProgram.initialized) { - [yuvConversionProgram addAttribute:@"position"]; - [yuvConversionProgram addAttribute:@"inputTextureCoordinate"]; + [self->yuvConversionProgram addAttribute:@"position"]; + [self->yuvConversionProgram addAttribute:@"inputTextureCoordinate"]; - if (![yuvConversionProgram link]) + if (![self->yuvConversionProgram link]) { - NSString *progLog = [yuvConversionProgram programLog]; + NSString *progLog = [self->yuvConversionProgram programLog]; NSLog(@"Program link log: %@", progLog); - NSString *fragLog = [yuvConversionProgram fragmentShaderLog]; + NSString *fragLog = [self->yuvConversionProgram fragmentShaderLog]; NSLog(@"Fragment shader compile log: %@", fragLog); - NSString *vertLog = [yuvConversionProgram vertexShaderLog]; + NSString *vertLog = [self->yuvConversionProgram vertexShaderLog]; NSLog(@"Vertex shader compile log: %@", vertLog); - yuvConversionProgram = nil; + self->yuvConversionProgram = nil; NSAssert(NO, @"Filter shader link failed"); } } - yuvConversionPositionAttribute = [yuvConversionProgram attributeIndex:@"position"]; - yuvConversionTextureCoordinateAttribute = [yuvConversionProgram attributeIndex:@"inputTextureCoordinate"]; - yuvConversionLuminanceTextureUniform = [yuvConversionProgram uniformIndex:@"luminanceTexture"]; - yuvConversionChrominanceTextureUniform = [yuvConversionProgram uniformIndex:@"chrominanceTexture"]; - yuvConversionMatrixUniform = [yuvConversionProgram uniformIndex:@"colorConversionMatrix"]; + self->yuvConversionPositionAttribute = [self->yuvConversionProgram attributeIndex:@"position"]; + self->yuvConversionTextureCoordinateAttribute = [self->yuvConversionProgram attributeIndex:@"inputTextureCoordinate"]; + self->yuvConversionLuminanceTextureUniform = [self->yuvConversionProgram uniformIndex:@"luminanceTexture"]; + self->yuvConversionChrominanceTextureUniform = [self->yuvConversionProgram uniformIndex:@"chrominanceTexture"]; + self->yuvConversionMatrixUniform = [self->yuvConversionProgram uniformIndex:@"colorConversionMatrix"]; - [GPUImageContext setActiveShaderProgram:yuvConversionProgram]; + [GPUImageContext setActiveShaderProgram:self->yuvConversionProgram]; - glEnableVertexAttribArray(yuvConversionPositionAttribute); - glEnableVertexAttribArray(yuvConversionTextureCoordinateAttribute); + glEnableVertexAttribArray(self->yuvConversionPositionAttribute); + glEnableVertexAttribArray(self->yuvConversionTextureCoordinateAttribute); } }); @@ -896,7 +896,7 @@ - (void)captureOutput:(AVCaptureOutput *)captureOutput didOutputSampleBuffer:(CM [self processVideoSampleBuffer:sampleBuffer]; CFRelease(sampleBuffer); - dispatch_semaphore_signal(frameRenderingSemaphore); + dispatch_semaphore_signal(self->frameRenderingSemaphore); }); } } @@ -925,56 +925,56 @@ - (void)updateOrientationSendToTargets; // From the iOS 5.0 release notes: // In previous iOS versions, the front-facing camera would always deliver buffers in AVCaptureVideoOrientationLandscapeLeft and the back-facing camera would always deliver buffers in AVCaptureVideoOrientationLandscapeRight. - if (captureAsYUV && [GPUImageContext supportsFastTextureUpload]) + if (self->captureAsYUV && [GPUImageContext supportsFastTextureUpload]) { - outputRotation = kGPUImageNoRotation; + self->outputRotation = kGPUImageNoRotation; if ([self cameraPosition] == AVCaptureDevicePositionBack) { - if (_horizontallyMirrorRearFacingCamera) + if (self->_horizontallyMirrorRearFacingCamera) { - switch(_outputImageOrientation) + switch(self->_outputImageOrientation) { - case UIInterfaceOrientationPortrait:internalRotation = kGPUImageRotateRightFlipVertical; break; - case UIInterfaceOrientationPortraitUpsideDown:internalRotation = kGPUImageRotate180; break; - case UIInterfaceOrientationLandscapeLeft:internalRotation = kGPUImageFlipHorizonal; break; - case UIInterfaceOrientationLandscapeRight:internalRotation = kGPUImageFlipVertical; break; - default:internalRotation = kGPUImageNoRotation; + case UIInterfaceOrientationPortrait:self->internalRotation = kGPUImageRotateRightFlipVertical; break; + case UIInterfaceOrientationPortraitUpsideDown:self->internalRotation = kGPUImageRotate180; break; + case UIInterfaceOrientationLandscapeLeft:self->internalRotation = kGPUImageFlipHorizonal; break; + case UIInterfaceOrientationLandscapeRight:self->internalRotation = kGPUImageFlipVertical; break; + default:self->internalRotation = kGPUImageNoRotation; } } else { - switch(_outputImageOrientation) + switch(self->_outputImageOrientation) { - case UIInterfaceOrientationPortrait:internalRotation = kGPUImageRotateRight; break; - case UIInterfaceOrientationPortraitUpsideDown:internalRotation = kGPUImageRotateLeft; break; - case UIInterfaceOrientationLandscapeLeft:internalRotation = kGPUImageRotate180; break; - case UIInterfaceOrientationLandscapeRight:internalRotation = kGPUImageNoRotation; break; - default:internalRotation = kGPUImageNoRotation; + case UIInterfaceOrientationPortrait:self->internalRotation = kGPUImageRotateRight; break; + case UIInterfaceOrientationPortraitUpsideDown:self->internalRotation = kGPUImageRotateLeft; break; + case UIInterfaceOrientationLandscapeLeft:self->internalRotation = kGPUImageRotate180; break; + case UIInterfaceOrientationLandscapeRight:self->internalRotation = kGPUImageNoRotation; break; + default:self->internalRotation = kGPUImageNoRotation; } } } else { - if (_horizontallyMirrorFrontFacingCamera) + if (self->_horizontallyMirrorFrontFacingCamera) { - switch(_outputImageOrientation) + switch(self->_outputImageOrientation) { - case UIInterfaceOrientationPortrait:internalRotation = kGPUImageRotateRightFlipVertical; break; - case UIInterfaceOrientationPortraitUpsideDown:internalRotation = kGPUImageRotateRightFlipHorizontal; break; - case UIInterfaceOrientationLandscapeLeft:internalRotation = kGPUImageFlipHorizonal; break; - case UIInterfaceOrientationLandscapeRight:internalRotation = kGPUImageFlipVertical; break; - default:internalRotation = kGPUImageNoRotation; + case UIInterfaceOrientationPortrait:self->internalRotation = kGPUImageRotateRightFlipVertical; break; + case UIInterfaceOrientationPortraitUpsideDown:self->internalRotation = kGPUImageRotateRightFlipHorizontal; break; + case UIInterfaceOrientationLandscapeLeft:self->internalRotation = kGPUImageFlipHorizonal; break; + case UIInterfaceOrientationLandscapeRight:self->internalRotation = kGPUImageFlipVertical; break; + default:self->internalRotation = kGPUImageNoRotation; } } else { - switch(_outputImageOrientation) + switch(self->_outputImageOrientation) { - case UIInterfaceOrientationPortrait:internalRotation = kGPUImageRotateRight; break; - case UIInterfaceOrientationPortraitUpsideDown:internalRotation = kGPUImageRotateLeft; break; - case UIInterfaceOrientationLandscapeLeft:internalRotation = kGPUImageNoRotation; break; - case UIInterfaceOrientationLandscapeRight:internalRotation = kGPUImageRotate180; break; - default:internalRotation = kGPUImageNoRotation; + case UIInterfaceOrientationPortrait:self->internalRotation = kGPUImageRotateRight; break; + case UIInterfaceOrientationPortraitUpsideDown:self->internalRotation = kGPUImageRotateLeft; break; + case UIInterfaceOrientationLandscapeLeft:self->internalRotation = kGPUImageNoRotation; break; + case UIInterfaceOrientationLandscapeRight:self->internalRotation = kGPUImageRotate180; break; + default:self->internalRotation = kGPUImageNoRotation; } } } @@ -983,60 +983,60 @@ - (void)updateOrientationSendToTargets; { if ([self cameraPosition] == AVCaptureDevicePositionBack) { - if (_horizontallyMirrorRearFacingCamera) + if (self->_horizontallyMirrorRearFacingCamera) { - switch(_outputImageOrientation) + switch(self->_outputImageOrientation) { - case UIInterfaceOrientationPortrait:outputRotation = kGPUImageRotateRightFlipVertical; break; - case UIInterfaceOrientationPortraitUpsideDown:outputRotation = kGPUImageRotate180; break; - case UIInterfaceOrientationLandscapeLeft:outputRotation = kGPUImageFlipHorizonal; break; - case UIInterfaceOrientationLandscapeRight:outputRotation = kGPUImageFlipVertical; break; - default:outputRotation = kGPUImageNoRotation; + case UIInterfaceOrientationPortrait:self->outputRotation = kGPUImageRotateRightFlipVertical; break; + case UIInterfaceOrientationPortraitUpsideDown:self->outputRotation = kGPUImageRotate180; break; + case UIInterfaceOrientationLandscapeLeft:self->outputRotation = kGPUImageFlipHorizonal; break; + case UIInterfaceOrientationLandscapeRight:self->outputRotation = kGPUImageFlipVertical; break; + default:self->outputRotation = kGPUImageNoRotation; } } else { - switch(_outputImageOrientation) + switch(self->_outputImageOrientation) { - case UIInterfaceOrientationPortrait:outputRotation = kGPUImageRotateRight; break; - case UIInterfaceOrientationPortraitUpsideDown:outputRotation = kGPUImageRotateLeft; break; - case UIInterfaceOrientationLandscapeLeft:outputRotation = kGPUImageRotate180; break; - case UIInterfaceOrientationLandscapeRight:outputRotation = kGPUImageNoRotation; break; - default:outputRotation = kGPUImageNoRotation; + case UIInterfaceOrientationPortrait:self->outputRotation = kGPUImageRotateRight; break; + case UIInterfaceOrientationPortraitUpsideDown:self->outputRotation = kGPUImageRotateLeft; break; + case UIInterfaceOrientationLandscapeLeft:self->outputRotation = kGPUImageRotate180; break; + case UIInterfaceOrientationLandscapeRight:self->outputRotation = kGPUImageNoRotation; break; + default:self->outputRotation = kGPUImageNoRotation; } } } else { - if (_horizontallyMirrorFrontFacingCamera) + if (self->_horizontallyMirrorFrontFacingCamera) { - switch(_outputImageOrientation) + switch(self->_outputImageOrientation) { - case UIInterfaceOrientationPortrait:outputRotation = kGPUImageRotateRightFlipVertical; break; - case UIInterfaceOrientationPortraitUpsideDown:outputRotation = kGPUImageRotateRightFlipHorizontal; break; - case UIInterfaceOrientationLandscapeLeft:outputRotation = kGPUImageFlipHorizonal; break; - case UIInterfaceOrientationLandscapeRight:outputRotation = kGPUImageFlipVertical; break; - default:outputRotation = kGPUImageNoRotation; + case UIInterfaceOrientationPortrait:self->outputRotation = kGPUImageRotateRightFlipVertical; break; + case UIInterfaceOrientationPortraitUpsideDown:self->outputRotation = kGPUImageRotateRightFlipHorizontal; break; + case UIInterfaceOrientationLandscapeLeft:self->outputRotation = kGPUImageFlipHorizonal; break; + case UIInterfaceOrientationLandscapeRight:self->outputRotation = kGPUImageFlipVertical; break; + default:self->outputRotation = kGPUImageNoRotation; } } else { - switch(_outputImageOrientation) + switch(self->_outputImageOrientation) { - case UIInterfaceOrientationPortrait:outputRotation = kGPUImageRotateRight; break; - case UIInterfaceOrientationPortraitUpsideDown:outputRotation = kGPUImageRotateLeft; break; - case UIInterfaceOrientationLandscapeLeft:outputRotation = kGPUImageNoRotation; break; - case UIInterfaceOrientationLandscapeRight:outputRotation = kGPUImageRotate180; break; - default:outputRotation = kGPUImageNoRotation; + case UIInterfaceOrientationPortrait:self->outputRotation = kGPUImageRotateRight; break; + case UIInterfaceOrientationPortraitUpsideDown:self->outputRotation = kGPUImageRotateLeft; break; + case UIInterfaceOrientationLandscapeLeft:self->outputRotation = kGPUImageNoRotation; break; + case UIInterfaceOrientationLandscapeRight:self->outputRotation = kGPUImageRotate180; break; + default:self->outputRotation = kGPUImageNoRotation; } } } } - for (id currentTarget in targets) + for (id currentTarget in self->targets) { - NSInteger indexOfObject = [targets indexOfObject:currentTarget]; - [currentTarget setInputRotation:outputRotation atIndex:[[targetTextureIndices objectAtIndex:indexOfObject] integerValue]]; + NSInteger indexOfObject = [self->targets indexOfObject:currentTarget]; + [currentTarget setInputRotation:self->outputRotation atIndex:[[self->targetTextureIndices objectAtIndex:indexOfObject] integerValue]]; } }); } diff --git a/ScannerApp/Frameworks/GPUImage/framework/Source/Mac/GPUImage.h b/ScannerApp/Frameworks/GPUImage/framework/Source/Mac/GPUImage.h deleted file mode 100755 index 0546740..0000000 --- a/ScannerApp/Frameworks/GPUImage/framework/Source/Mac/GPUImage.h +++ /dev/null @@ -1,167 +0,0 @@ -#import - -// Base classes -#import -#import - -// Sources -#import -#import -#import -#import -#import -#import -#import -#import - -// Filters -#import -#import -#import -#import -#import -#import -#import -#import -#import -#import -#import -#import -#import -#import -#import -#import -#import -#import -#import -#import -#import -#import -#import -#import -#import -#import -#import -#import -#import -#import -#import -#import -#import -#import -#import -#import -#import -#import -#import -#import -#import -#import -#import -#import -#import -#import -#import -#import -#import -#import -#import -#import -#import -#import -#import -#import -#import -#import -#import -#import -#import -#import -#import -#import -#import -#import -#import -#import -#import -#import -#import -#import -#import -#import -#import -#import -#import -#import -#import -#import -#import -#import -#import -#import -#import -#import -#import -#import -#import -#import -#import -#import -#import -#import -#import -#import -#import -#import -#import -#import -#import -#import -#import -#import -#import -#import -#import -#import -#import -#import -#import -#import -#import -#import -#import -#import -#import -#import -#import -#import -#import -#import -#import -#import -#import -#import -#import -#import -#import -#import -#import -#import -#import -#import -#import -#import -#import -#import -#import -#import -#import -#import -#import -#import -#import - -// Outputs -#import -#import -#import diff --git a/ScannerApp/Frameworks/GPUImage/framework/Source/Mac/GPUImageAVCamera.h b/ScannerApp/Frameworks/GPUImage/framework/Source/Mac/GPUImageAVCamera.h deleted file mode 100755 index 54c1a2f..0000000 --- a/ScannerApp/Frameworks/GPUImage/framework/Source/Mac/GPUImageAVCamera.h +++ /dev/null @@ -1,132 +0,0 @@ -#import -#import -#import -#import "GPUImageContext.h" -#import "GPUImageOutput.h" - -//Delegate Protocal for Face Detection. -@protocol GPUImageVideoCameraDelegate - -@optional -- (void)willOutputSampleBuffer:(CMSampleBufferRef)sampleBuffer; -@end - - -/** - A GPUImageOutput that provides frames from either camera -*/ -@interface GPUImageAVCamera : GPUImageOutput -{ - NSUInteger numberOfFramesCaptured; - CGFloat totalFrameTimeDuringCapture; - - AVCaptureSession *_captureSession; - AVCaptureDevice *_inputCamera; - AVCaptureDevice *_microphone; - AVCaptureDeviceInput *videoInput; - AVCaptureVideoDataOutput *videoOutput; - - BOOL capturePaused; - GPUImageRotationMode outputRotation; - dispatch_semaphore_t frameRenderingSemaphore; - - BOOL captureAsYUV; - GLuint luminanceTexture, chrominanceTexture; - - __unsafe_unretained id _delegate; -} - -/// The AVCaptureSession used to capture from the camera -@property(readonly, retain, nonatomic) AVCaptureSession *captureSession; - -/// This enables the capture session preset to be changed on the fly -@property (readwrite, nonatomic, copy) NSString *captureSessionPreset; - -/// This sets the frame rate of the camera (iOS 5 and above only) -/** - Setting this to 0 or below will set the frame rate back to the default setting for a particular preset. - */ -@property (readwrite) NSInteger frameRate; - -/// Easy way to tell if front-facing camera is present on device -@property (readonly, getter = isFrontFacingCameraPresent) BOOL frontFacingCameraPresent; - -/// This enables the benchmarking mode, which logs out instantaneous and average frame times to the console -@property(readwrite, nonatomic) BOOL runBenchmark; - -/// Use this property to manage camera settings. Focus point, exposure point, etc. -@property(readonly) AVCaptureDevice *inputCamera; - -/// These properties determine whether or not the two camera orientations should be mirrored. By default, both are NO. -@property(readwrite, nonatomic) BOOL horizontallyMirrorFrontFacingCamera, horizontallyMirrorRearFacingCamera; - -@property(nonatomic, assign) id delegate; - -/// @name Initialization and teardown - -+ (NSArray *)connectedCameraDevices; - -/** Begin a capture session - - See AVCaptureSession for acceptable values - - @param sessionPreset Session preset to use - @param cameraPosition Camera to capture from - */ -- (id)initWithDeviceUniqueID:(NSString *)deviceUniqueID; -- (id)initWithSessionPreset:(NSString *)sessionPreset deviceUniqueID:(NSString *)deviceUniqueID; -- (id)initWithSessionPreset:(NSString *)sessionPreset cameraDevice:(AVCaptureDevice *)cameraDevice; - -/** Tear down the capture session - */ -- (void)removeInputsAndOutputs; - -/// @name Manage the camera video stream - -/** Start camera capturing - */ -- (void)startCameraCapture; - -/** Stop camera capturing - */ -- (void)stopCameraCapture; - -/** Pause camera capturing - */ -- (void)pauseCameraCapture; - -/** Resume camera capturing - */ -- (void)resumeCameraCapture; - -/** Process a video sample - @param sampleBuffer Buffer to process - */ -- (void)processVideoSampleBuffer:(CMSampleBufferRef)sampleBuffer; - -/** Process an audio sample - @param sampleBuffer Buffer to process - */ -- (void)processAudioSampleBuffer:(CMSampleBufferRef)sampleBuffer; - -/** Get the position (front, rear) of the source camera - */ -- (AVCaptureDevicePosition)cameraPosition; - -/** Get the AVCaptureConnection of the source camera - */ -- (AVCaptureConnection *)videoCaptureConnection; - -/** This flips between the front and rear cameras - */ -- (void)rotateCamera; - -/// @name Benchmarking - -/** When benchmarking is enabled, this will keep a running average of the time from uploading, processing, and final recording or display - */ -- (CGFloat)averageFrameDurationDuringCapture; - -- (void)printSupportedPixelFormats; - -@end diff --git a/ScannerApp/Frameworks/GPUImage/framework/Source/Mac/GPUImageAVCamera.m b/ScannerApp/Frameworks/GPUImage/framework/Source/Mac/GPUImageAVCamera.m deleted file mode 100644 index 4b26c75..0000000 --- a/ScannerApp/Frameworks/GPUImage/framework/Source/Mac/GPUImageAVCamera.m +++ /dev/null @@ -1,771 +0,0 @@ -#import "GPUImageAVCamera.h" -#import "GPUImageMovieWriter.h" -#import "GPUImageFilter.h" -#import "GPUImageColorConversion.h" - -#pragma mark - -#pragma mark Private methods and instance variables - -@interface GPUImageAVCamera () -{ - AVCaptureDeviceInput *audioInput; - AVCaptureAudioDataOutput *audioOutput; - NSDate *startingCaptureTime; - - NSInteger _frameRate; - - dispatch_queue_t cameraProcessingQueue, audioProcessingQueue; - - GLProgram *yuvConversionProgram; - GLint yuvConversionPositionAttribute, yuvConversionTextureCoordinateAttribute; - GLint yuvConversionLuminanceTextureUniform, yuvConversionChrominanceTextureUniform; - - int imageBufferWidth, imageBufferHeight; -} - -- (void)updateOrientationSendToTargets; -- (void)convertYUVToRGBOutput; - -@end - -@implementation GPUImageAVCamera - -@synthesize captureSessionPreset = _captureSessionPreset; -@synthesize captureSession = _captureSession; -@synthesize inputCamera = _inputCamera; -@synthesize runBenchmark = _runBenchmark; -@synthesize delegate = _delegate; -@synthesize horizontallyMirrorFrontFacingCamera = _horizontallyMirrorFrontFacingCamera, horizontallyMirrorRearFacingCamera = _horizontallyMirrorRearFacingCamera; - -#pragma mark - -#pragma mark Initialization and teardown - -+ (NSArray *)connectedCameraDevices; -{ - NSArray *devices = [AVCaptureDevice devicesWithMediaType:AVMediaTypeVideo]; - return devices; -} - -- (id)init; -{ - if (!(self = [self initWithSessionPreset:AVCaptureSessionPreset640x480 cameraDevice:nil])) - { - return nil; - } - - return self; -} - -- (id)initWithDeviceUniqueID:(NSString *)deviceUniqueID; -{ - if (!(self = [self initWithSessionPreset:AVCaptureSessionPreset640x480 deviceUniqueID:deviceUniqueID])) - { - return nil; - } - - return self; -} - -- (id)initWithSessionPreset:(NSString *)sessionPreset deviceUniqueID:(NSString *)deviceUniqueID; -{ - if (!(self = [self initWithSessionPreset:sessionPreset cameraDevice:[AVCaptureDevice deviceWithUniqueID:deviceUniqueID]])) - { - return nil; - } - - return self; -} - -- (id)initWithSessionPreset:(NSString *)sessionPreset cameraDevice:(AVCaptureDevice *)cameraDevice; -{ - if (!(self = [super init])) - { - return nil; - } - - cameraProcessingQueue = dispatch_queue_create("com.sunsetlakesoftware.GPUImage.cameraProcessingQueue", NULL); - audioProcessingQueue = dispatch_queue_create("com.sunsetlakesoftware.GPUImage.audioProcessingQueue", NULL); - frameRenderingSemaphore = dispatch_semaphore_create(1); - - _frameRate = 0; // This will not set frame rate unless this value gets set to 1 or above - _runBenchmark = NO; - capturePaused = NO; - outputRotation = kGPUImageNoRotation; -// captureAsYUV = YES; - captureAsYUV = NO; - - runSynchronouslyOnVideoProcessingQueue(^{ - - if (captureAsYUV) - { - [GPUImageContext useImageProcessingContext]; -// if ([GPUImageContext deviceSupportsRedTextures]) -// { -// yuvConversionProgram = [[GPUImageContext sharedImageProcessingContext] programForVertexShaderString:kGPUImageVertexShaderString fragmentShaderString:kGPUImageYUVVideoRangeConversionForRGFragmentShaderString]; -// } -// else -// { - yuvConversionProgram = [[GPUImageContext sharedImageProcessingContext] programForVertexShaderString:kGPUImageVertexShaderString fragmentShaderString:kGPUImageYUVVideoRangeConversionForLAFragmentShaderString]; -// } - - if (!yuvConversionProgram.initialized) - { - [yuvConversionProgram addAttribute:@"position"]; - [yuvConversionProgram addAttribute:@"inputTextureCoordinate"]; - - if (![yuvConversionProgram link]) - { - NSString *progLog = [yuvConversionProgram programLog]; - NSLog(@"Program link log: %@", progLog); - NSString *fragLog = [yuvConversionProgram fragmentShaderLog]; - NSLog(@"Fragment shader compile log: %@", fragLog); - NSString *vertLog = [yuvConversionProgram vertexShaderLog]; - NSLog(@"Vertex shader compile log: %@", vertLog); - yuvConversionProgram = nil; - NSAssert(NO, @"Filter shader link failed"); - } - } - - yuvConversionPositionAttribute = [yuvConversionProgram attributeIndex:@"position"]; - yuvConversionTextureCoordinateAttribute = [yuvConversionProgram attributeIndex:@"inputTextureCoordinate"]; - yuvConversionLuminanceTextureUniform = [yuvConversionProgram uniformIndex:@"luminanceTexture"]; - yuvConversionChrominanceTextureUniform = [yuvConversionProgram uniformIndex:@"chrominanceTexture"]; - - [GPUImageContext setActiveShaderProgram:yuvConversionProgram]; - - glEnableVertexAttribArray(yuvConversionPositionAttribute); - glEnableVertexAttribArray(yuvConversionTextureCoordinateAttribute); - } - }); - - // Grab the back-facing or front-facing camera - _inputCamera = nil; - - if (cameraDevice == nil) - { - _inputCamera = [AVCaptureDevice defaultDeviceWithMediaType:AVMediaTypeVideo]; - } - else - { - _inputCamera = cameraDevice; - } - - if (!_inputCamera) { - return nil; - } - - // Create the capture session - _captureSession = [[AVCaptureSession alloc] init]; - - [_captureSession beginConfiguration]; - - // Add the video input - NSError *error = nil; - videoInput = [[AVCaptureDeviceInput alloc] initWithDevice:_inputCamera error:&error]; - if ([_captureSession canAddInput:videoInput]) - { - [_captureSession addInput:videoInput]; - } - - // Add the video frame output - videoOutput = [[AVCaptureVideoDataOutput alloc] init]; - [videoOutput setAlwaysDiscardsLateVideoFrames:NO]; - -// NSLog(@"Camera: %@", _inputCamera); -// [self printSupportedPixelFormats]; - -// if (captureAsYUV && [GPUImageContext deviceSupportsRedTextures]) - if (captureAsYUV && [GPUImageContext supportsFastTextureUpload]) - { - BOOL supportsFullYUVRange = NO; - NSArray *supportedPixelFormats = videoOutput.availableVideoCVPixelFormatTypes; - for (NSNumber *currentPixelFormat in supportedPixelFormats) - { - if ([currentPixelFormat intValue] == kCVPixelFormatType_420YpCbCr8BiPlanarFullRange) - { - supportsFullYUVRange = YES; - } - } - - if (supportsFullYUVRange) - { - [videoOutput setVideoSettings:[NSDictionary dictionaryWithObject:[NSNumber numberWithInt:kCVPixelFormatType_420YpCbCr8BiPlanarFullRange] forKey:(id)kCVPixelBufferPixelFormatTypeKey]]; - } - else - { - [videoOutput setVideoSettings:[NSDictionary dictionaryWithObject:[NSNumber numberWithInt:kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange] forKey:(id)kCVPixelBufferPixelFormatTypeKey]]; - } - } - else - { - // Despite returning a longer list of supported pixel formats, only RGB, RGBA, BGRA, and the YUV 4:2:2 variants seem to return cleanly - [videoOutput setVideoSettings:[NSDictionary dictionaryWithObject:[NSNumber numberWithInt:kCVPixelFormatType_32BGRA] forKey:(id)kCVPixelBufferPixelFormatTypeKey]]; -// [videoOutput setVideoSettings:[NSDictionary dictionaryWithObject:[NSNumber numberWithInt:kCVPixelFormatType_422YpCbCr8_yuvs] forKey:(id)kCVPixelBufferPixelFormatTypeKey]]; - } - - [videoOutput setSampleBufferDelegate:self queue:cameraProcessingQueue]; -// [videoOutput setSampleBufferDelegate:self queue:[GPUImageContext sharedContextQueue]]; - if ([_captureSession canAddOutput:videoOutput]) - { - [_captureSession addOutput:videoOutput]; - } - else - { - NSLog(@"Couldn't add video output"); - return nil; - } - - _captureSessionPreset = sessionPreset; - [_captureSession setSessionPreset:_captureSessionPreset]; - -// This will let you get 60 FPS video from the 720p preset on an iPhone 4S, but only that device and that preset -// AVCaptureConnection *conn = [videoOutput connectionWithMediaType:AVMediaTypeVideo]; -// -// if (conn.supportsVideoMinFrameDuration) -// conn.videoMinFrameDuration = CMTimeMake(1,60); -// if (conn.supportsVideoMaxFrameDuration) -// conn.videoMaxFrameDuration = CMTimeMake(1,60); - - [_captureSession commitConfiguration]; - - return self; -} - -- (void)dealloc -{ - [self stopCameraCapture]; - [videoOutput setSampleBufferDelegate:nil queue:dispatch_get_main_queue()]; - [audioOutput setSampleBufferDelegate:nil queue:dispatch_get_main_queue()]; - - [self removeInputsAndOutputs]; - -// ARC forbids explicit message send of 'release'; since iOS 6 even for dispatch_release() calls: stripping it out in that case is required. -//#if ( (__IPHONE_OS_VERSION_MIN_REQUIRED < __IPHONE_6_0) || (!defined(__IPHONE_6_0)) ) -#if __MAC_OS_X_VERSION_MAX_ALLOWED <= __MAC_10_7 - if (cameraProcessingQueue != NULL) - { - dispatch_release(cameraProcessingQueue); - } - - if (audioProcessingQueue != NULL) - { - dispatch_release(audioProcessingQueue); - } - - if (frameRenderingSemaphore != NULL) - { - dispatch_release(frameRenderingSemaphore); - } -#endif -} - -- (void)removeInputsAndOutputs; -{ - [_captureSession removeInput:videoInput]; - [_captureSession removeOutput:videoOutput]; - if (_microphone != nil) - { - [_captureSession removeInput:audioInput]; - [_captureSession removeOutput:audioOutput]; - } -} - -#pragma mark - -#pragma mark Managing targets - -- (void)addTarget:(id)newTarget atTextureLocation:(NSInteger)textureLocation; -{ - [super addTarget:newTarget atTextureLocation:textureLocation]; - - [newTarget setInputRotation:outputRotation atIndex:textureLocation]; -} - -#pragma mark - -#pragma mark Manage the camera video stream - -- (void)startCameraCapture; -{ - if (![_captureSession isRunning]) - { - startingCaptureTime = [NSDate date]; - [_captureSession startRunning]; - }; -} - -- (void)stopCameraCapture; -{ - if ([_captureSession isRunning]) - { - [_captureSession stopRunning]; - } -} - -- (void)pauseCameraCapture; -{ - capturePaused = YES; -} - -- (void)resumeCameraCapture; -{ - capturePaused = NO; -} - -- (void)rotateCamera -{ - if (self.frontFacingCameraPresent == NO) - return; - - NSError *error; - AVCaptureDeviceInput *newVideoInput; - AVCaptureDevicePosition currentCameraPosition = [[videoInput device] position]; - - if (currentCameraPosition == AVCaptureDevicePositionBack) - { - currentCameraPosition = AVCaptureDevicePositionFront; - } - else - { - currentCameraPosition = AVCaptureDevicePositionBack; - } - - AVCaptureDevice *backFacingCamera = nil; - NSArray *devices = [AVCaptureDevice devicesWithMediaType:AVMediaTypeVideo]; - for (AVCaptureDevice *device in devices) - { - if ([device position] == currentCameraPosition) - { - backFacingCamera = device; - } - } - newVideoInput = [[AVCaptureDeviceInput alloc] initWithDevice:backFacingCamera error:&error]; - - if (newVideoInput != nil) - { - [_captureSession beginConfiguration]; - - [_captureSession removeInput:videoInput]; - if ([_captureSession canAddInput:newVideoInput]) - { - [_captureSession addInput:newVideoInput]; - videoInput = newVideoInput; - } - else - { - [_captureSession addInput:videoInput]; - } - //captureSession.sessionPreset = oriPreset; - [_captureSession commitConfiguration]; - } - - _inputCamera = backFacingCamera; -} - -- (AVCaptureDevicePosition)cameraPosition -{ - return [[videoInput device] position]; -} - -- (BOOL)isFrontFacingCameraPresent; -{ - NSArray *devices = [AVCaptureDevice devicesWithMediaType:AVMediaTypeVideo]; - - for (AVCaptureDevice *device in devices) - { - if ([device position] == AVCaptureDevicePositionFront) - return YES; - } - - return NO; -} - -- (void)setCaptureSessionPreset:(NSString *)captureSessionPreset; -{ - [_captureSession beginConfiguration]; - - _captureSessionPreset = captureSessionPreset; - [_captureSession setSessionPreset:_captureSessionPreset]; - - [_captureSession commitConfiguration]; -} - -- (void)setFrameRate:(NSInteger)frameRate; -{ - _frameRate = frameRate; - - if (_frameRate > 0) - { - for (AVCaptureConnection *connection in videoOutput.connections) - { - if ([connection respondsToSelector:@selector(setVideoMinFrameDuration:)]) - connection.videoMinFrameDuration = CMTimeMake(1, (int32_t)_frameRate); - - } - } - else - { - for (AVCaptureConnection *connection in videoOutput.connections) - { - if ([connection respondsToSelector:@selector(setVideoMinFrameDuration:)]) - connection.videoMinFrameDuration = kCMTimeInvalid; // This sets videoMinFrameDuration back to default - } - } -} - -- (NSInteger)frameRate; -{ - return _frameRate; -} - -- (AVCaptureConnection *)videoCaptureConnection { - for (AVCaptureConnection *connection in [videoOutput connections] ) { - for ( AVCaptureInputPort *port in [connection inputPorts] ) { - if ( [[port mediaType] isEqual:AVMediaTypeVideo] ) { - return connection; - } - } - } - - return nil; -} - -#define INITIALFRAMESTOIGNOREFORBENCHMARK 5 - -- (void)updateTargetsForVideoCameraUsingCacheTextureAtWidth:(int)bufferWidth height:(int)bufferHeight time:(CMTime)currentTime; -{ - // First, update all the framebuffers in the targets - for (id currentTarget in targets) - { - if ([currentTarget enabled]) - { - NSInteger indexOfObject = [targets indexOfObject:currentTarget]; - NSInteger textureIndexOfTarget = [[targetTextureIndices objectAtIndex:indexOfObject] integerValue]; - - if (currentTarget != self.targetToIgnoreForUpdates) - { - [currentTarget setInputRotation:outputRotation atIndex:textureIndexOfTarget]; - [currentTarget setInputSize:CGSizeMake(bufferWidth, bufferHeight) atIndex:textureIndexOfTarget]; - - if ([currentTarget wantsMonochromeInput] && captureAsYUV) - { - [currentTarget setCurrentlyReceivingMonochromeInput:YES]; - // TODO: Replace optimization for monochrome output - [currentTarget setInputFramebuffer:outputFramebuffer atIndex:textureIndexOfTarget]; - } - else - { - [currentTarget setCurrentlyReceivingMonochromeInput:NO]; - [currentTarget setInputFramebuffer:outputFramebuffer atIndex:textureIndexOfTarget]; - } - } - else - { - [currentTarget setInputRotation:outputRotation atIndex:textureIndexOfTarget]; - [currentTarget setInputFramebuffer:outputFramebuffer atIndex:textureIndexOfTarget]; - } - } - } - - // Then release our hold on the local framebuffer to send it back to the cache as soon as it's no longer needed - [outputFramebuffer unlock]; - - // Finally, trigger rendering as needed - for (id currentTarget in targets) - { - if ([currentTarget enabled]) - { - NSInteger indexOfObject = [targets indexOfObject:currentTarget]; - NSInteger textureIndexOfTarget = [[targetTextureIndices objectAtIndex:indexOfObject] integerValue]; - - if (currentTarget != self.targetToIgnoreForUpdates) - { - [currentTarget newFrameReadyAtTime:currentTime atIndex:textureIndexOfTarget]; - } - } - } -} - -- (void)processVideoSampleBuffer:(CMSampleBufferRef)sampleBuffer; -{ - if (capturePaused) - { - return; - } - - CFAbsoluteTime startTime = CFAbsoluteTimeGetCurrent(); - CVImageBufferRef cameraFrame = CMSampleBufferGetImageBuffer(sampleBuffer); - GLsizei bufferWidth = (GLsizei)CVPixelBufferGetWidth(cameraFrame); - GLsizei bufferHeight = (GLsizei)CVPixelBufferGetHeight(cameraFrame); - - CMTime currentTime = CMSampleBufferGetPresentationTimeStamp(sampleBuffer); - - [GPUImageContext useImageProcessingContext]; - - CVPixelBufferLockBaseAddress(cameraFrame, 0); - - outputFramebuffer = [[GPUImageContext sharedFramebufferCache] fetchFramebufferForSize:CGSizeMake(bufferWidth, bufferHeight) onlyTexture:YES]; - - glBindTexture(GL_TEXTURE_2D, [outputFramebuffer texture]); - - // glTexImage2D(GL_TEXTURE_2D, 0, GL_RGBA, bufferWidth, bufferHeight, 0, GL_BGRA, GL_UNSIGNED_BYTE, CVPixelBufferGetBaseAddress(cameraFrame)); - - // Using BGRA extension to pull in video frame data directly -// glTexImage2D(GL_TEXTURE_2D, 0, GL_RGB, bytesPerRow / 3, bufferHeight, 0, GL_RGB, GL_UNSIGNED_BYTE, CVPixelBufferGetBaseAddress(cameraFrame)); -// glTexImage2D(GL_TEXTURE_2D, 0, GL_RGBA, bufferWidth, bufferHeight, 0, GL_YCBCR_422_APPLE, GL_UNSIGNED_SHORT_8_8_REV_APPLE, CVPixelBufferGetBaseAddress(cameraFrame)); - glTexImage2D(GL_TEXTURE_2D, 0, GL_RGBA, bufferWidth, bufferHeight, 0, GL_BGRA, GL_UNSIGNED_BYTE, CVPixelBufferGetBaseAddress(cameraFrame)); - - [self updateTargetsForVideoCameraUsingCacheTextureAtWidth:bufferWidth height:bufferHeight time:currentTime]; - -// for (id currentTarget in targets) -// { -// if ([currentTarget enabled]) -// { -// if (currentTarget != self.targetToIgnoreForUpdates) -// { -// NSInteger indexOfObject = [targets indexOfObject:currentTarget]; -// NSInteger textureIndexOfTarget = [[targetTextureIndices objectAtIndex:indexOfObject] integerValue]; -// -// [currentTarget setInputSize:CGSizeMake(bufferWidth, bufferHeight) atIndex:textureIndexOfTarget]; -// [currentTarget newFrameReadyAtTime:currentTime atIndex:textureIndexOfTarget]; -// } -// } -// } - - CVPixelBufferUnlockBaseAddress(cameraFrame, 0); - - if (_runBenchmark) - { - numberOfFramesCaptured++; - if (numberOfFramesCaptured > INITIALFRAMESTOIGNOREFORBENCHMARK) - { - CFAbsoluteTime currentFrameTime = (CFAbsoluteTimeGetCurrent() - startTime); - totalFrameTimeDuringCapture += currentFrameTime; - NSLog(@"Average frame time : %f ms", [self averageFrameDurationDuringCapture]); - NSLog(@"Current frame time : %f ms", 1000.0 * currentFrameTime); - } - } -} - -- (void)processAudioSampleBuffer:(CMSampleBufferRef)sampleBuffer; -{ - [self.audioEncodingTarget processAudioBuffer:sampleBuffer]; -} - -- (void)convertYUVToRGBOutput; -{ - [GPUImageContext setActiveShaderProgram:yuvConversionProgram]; - - outputFramebuffer = [[GPUImageContext sharedFramebufferCache] fetchFramebufferForSize:CGSizeMake(imageBufferWidth, imageBufferHeight) textureOptions:self.outputTextureOptions onlyTexture:NO]; - [outputFramebuffer activateFramebuffer]; - - glClearColor(0.0f, 0.0f, 0.0f, 1.0f); - glClear(GL_COLOR_BUFFER_BIT | GL_DEPTH_BUFFER_BIT); - - static const GLfloat squareVertices[] = { - -1.0f, -1.0f, - 1.0f, -1.0f, - -1.0f, 1.0f, - 1.0f, 1.0f, - }; - - static const GLfloat textureCoordinates[] = { - 0.0f, 0.0f, - 1.0f, 0.0f, - 0.0f, 1.0f, - 1.0f, 1.0f, - }; - - glActiveTexture(GL_TEXTURE4); - glBindTexture(GL_TEXTURE_2D, luminanceTexture); - glUniform1i(yuvConversionLuminanceTextureUniform, 4); - - glActiveTexture(GL_TEXTURE5); - glBindTexture(GL_TEXTURE_2D, chrominanceTexture); - glUniform1i(yuvConversionChrominanceTextureUniform, 5); - - glVertexAttribPointer(yuvConversionPositionAttribute, 2, GL_FLOAT, 0, 0, squareVertices); - glVertexAttribPointer(yuvConversionTextureCoordinateAttribute, 2, GL_FLOAT, 0, 0, textureCoordinates); - - glDrawArrays(GL_TRIANGLE_STRIP, 0, 4); -} - -#pragma mark - -#pragma mark Benchmarking - -- (CGFloat)averageFrameDurationDuringCapture; -{ - return (totalFrameTimeDuringCapture / (CGFloat)(numberOfFramesCaptured - INITIALFRAMESTOIGNOREFORBENCHMARK)) * 1000.0; -} - -#pragma mark - -#pragma mark AVCaptureVideoDataOutputSampleBufferDelegate - -- (void)captureOutput:(AVCaptureOutput *)captureOutput didOutputSampleBuffer:(CMSampleBufferRef)sampleBuffer fromConnection:(AVCaptureConnection *)connection -{ - if (captureOutput == audioOutput) - { -// if (dispatch_semaphore_wait(frameRenderingSemaphore, DISPATCH_TIME_NOW) != 0) -// { -// return; -// } - - CFRetain(sampleBuffer); - runAsynchronouslyOnVideoProcessingQueue(^{ - [self processAudioSampleBuffer:sampleBuffer]; - CFRelease(sampleBuffer); -// dispatch_semaphore_signal(frameRenderingSemaphore); - }); - } - else - { - if (dispatch_semaphore_wait(frameRenderingSemaphore, DISPATCH_TIME_NOW) != 0) - { - return; - } - - CFRetain(sampleBuffer); - runAsynchronouslyOnVideoProcessingQueue(^{ - //Feature Detection Hook. - if (self.delegate && [self.delegate respondsToSelector:@selector(willOutputSampleBuffer:)]) - { - [self.delegate willOutputSampleBuffer:sampleBuffer]; - } - - [self processVideoSampleBuffer:sampleBuffer]; - - CFRelease(sampleBuffer); - dispatch_semaphore_signal(frameRenderingSemaphore); - }); - } -} - -#pragma mark - -#pragma mark Accessors - -- (void)setAudioEncodingTarget:(GPUImageMovieWriter *)newValue; -{ - runSynchronouslyOnVideoProcessingQueue(^{ - [_captureSession beginConfiguration]; - - if (newValue == nil) - { - if (audioOutput) - { - [_captureSession removeInput:audioInput]; - [_captureSession removeOutput:audioOutput]; - audioInput = nil; - audioOutput = nil; - _microphone = nil; - } - } - else - { - _microphone = [AVCaptureDevice defaultDeviceWithMediaType:AVMediaTypeAudio]; - audioInput = [AVCaptureDeviceInput deviceInputWithDevice:_microphone error:nil]; - if ([_captureSession canAddInput:audioInput]) - { - [_captureSession addInput:audioInput]; - } - audioOutput = [[AVCaptureAudioDataOutput alloc] init]; - - if ([_captureSession canAddOutput:audioOutput]) - { - [_captureSession addOutput:audioOutput]; - } - else - { - NSLog(@"Couldn't add audio output"); - } - [audioOutput setSampleBufferDelegate:self queue:audioProcessingQueue]; - } - - [_captureSession commitConfiguration]; - - [super setAudioEncodingTarget:newValue]; - }); -} - -- (void)updateOrientationSendToTargets; -{ - runSynchronouslyOnVideoProcessingQueue(^{ - - // From the iOS 5.0 release notes: - // In previous iOS versions, the front-facing camera would always deliver buffers in AVCaptureVideoOrientationLandscapeLeft and the back-facing camera would always deliver buffers in AVCaptureVideoOrientationLandscapeRight. - - outputRotation = kGPUImageNoRotation; - for (id currentTarget in targets) - { - NSInteger indexOfObject = [targets indexOfObject:currentTarget]; - [currentTarget setInputRotation:outputRotation atIndex:[[targetTextureIndices objectAtIndex:indexOfObject] integerValue]]; - } - }); -} - -- (void)setHorizontallyMirrorFrontFacingCamera:(BOOL)newValue -{ - _horizontallyMirrorFrontFacingCamera = newValue; - [self updateOrientationSendToTargets]; -} - -- (void)setHorizontallyMirrorRearFacingCamera:(BOOL)newValue -{ - _horizontallyMirrorRearFacingCamera = newValue; - [self updateOrientationSendToTargets]; -} - -- (void)printSupportedPixelFormats; -{ - NSArray *supportedPixelFormats = videoOutput.availableVideoCVPixelFormatTypes; - for (NSNumber *currentPixelFormat in supportedPixelFormats) - { - NSString *pixelFormatName = nil; - - switch([currentPixelFormat intValue]) - { - case kCVPixelFormatType_1Monochrome: pixelFormatName = @"kCVPixelFormatType_1Monochrome"; break; - case kCVPixelFormatType_2Indexed: pixelFormatName = @"kCVPixelFormatType_2Indexed"; break; - case kCVPixelFormatType_4Indexed: pixelFormatName = @"kCVPixelFormatType_4Indexed"; break; - case kCVPixelFormatType_8Indexed: pixelFormatName = @"kCVPixelFormatType_8Indexed"; break; - case kCVPixelFormatType_1IndexedGray_WhiteIsZero: pixelFormatName = @"kCVPixelFormatType_1IndexedGray_WhiteIsZero"; break; - case kCVPixelFormatType_2IndexedGray_WhiteIsZero: pixelFormatName = @"kCVPixelFormatType_2IndexedGray_WhiteIsZero"; break; - case kCVPixelFormatType_4IndexedGray_WhiteIsZero: pixelFormatName = @"kCVPixelFormatType_4IndexedGray_WhiteIsZero"; break; - case kCVPixelFormatType_8IndexedGray_WhiteIsZero: pixelFormatName = @"kCVPixelFormatType_8IndexedGray_WhiteIsZero"; break; - case kCVPixelFormatType_16BE555: pixelFormatName = @"kCVPixelFormatType_16BE555"; break; - case kCVPixelFormatType_16LE555: pixelFormatName = @"kCVPixelFormatType_16LE555"; break; - case kCVPixelFormatType_16LE5551: pixelFormatName = @"kCVPixelFormatType_16LE5551"; break; - case kCVPixelFormatType_16BE565: pixelFormatName = @"kCVPixelFormatType_16BE565"; break; - case kCVPixelFormatType_16LE565: pixelFormatName = @"kCVPixelFormatType_16LE565"; break; - case kCVPixelFormatType_24RGB: pixelFormatName = @"kCVPixelFormatType_24RGB"; break; - case kCVPixelFormatType_24BGR: pixelFormatName = @"kCVPixelFormatType_24BGR"; break; - case kCVPixelFormatType_32ARGB: pixelFormatName = @"kCVPixelFormatType_32ARGB"; break; - case kCVPixelFormatType_32BGRA: pixelFormatName = @"kCVPixelFormatType_32BGRA"; break; - case kCVPixelFormatType_32ABGR: pixelFormatName = @"kCVPixelFormatType_32ABGR"; break; - case kCVPixelFormatType_32RGBA: pixelFormatName = @"kCVPixelFormatType_32RGBA"; break; - case kCVPixelFormatType_64ARGB: pixelFormatName = @"kCVPixelFormatType_64ARGB"; break; - case kCVPixelFormatType_48RGB: pixelFormatName = @"kCVPixelFormatType_48RGB"; break; - case kCVPixelFormatType_32AlphaGray: pixelFormatName = @"kCVPixelFormatType_32AlphaGray"; break; - case kCVPixelFormatType_16Gray: pixelFormatName = @"kCVPixelFormatType_16Gray"; break; - case kCVPixelFormatType_30RGB: pixelFormatName = @"kCVPixelFormatType_30RGB"; break; - case kCVPixelFormatType_422YpCbCr8: pixelFormatName = @"kCVPixelFormatType_422YpCbCr8"; break; - case kCVPixelFormatType_4444YpCbCrA8: pixelFormatName = @"kCVPixelFormatType_4444YpCbCrA8"; break; - case kCVPixelFormatType_4444YpCbCrA8R: pixelFormatName = @"kCVPixelFormatType_4444YpCbCrA8R"; break; - case kCVPixelFormatType_4444AYpCbCr8: pixelFormatName = @"kCVPixelFormatType_4444AYpCbCr8"; break; - case kCVPixelFormatType_4444AYpCbCr16: pixelFormatName = @"kCVPixelFormatType_4444AYpCbCr16"; break; - case kCVPixelFormatType_444YpCbCr8: pixelFormatName = @"kCVPixelFormatType_444YpCbCr8"; break; - case kCVPixelFormatType_422YpCbCr16: pixelFormatName = @"kCVPixelFormatType_422YpCbCr16"; break; - case kCVPixelFormatType_422YpCbCr10: pixelFormatName = @"kCVPixelFormatType_422YpCbCr10"; break; - case kCVPixelFormatType_444YpCbCr10: pixelFormatName = @"kCVPixelFormatType_444YpCbCr10"; break; - case kCVPixelFormatType_420YpCbCr8Planar: pixelFormatName = @"kCVPixelFormatType_420YpCbCr8Planar"; break; - case kCVPixelFormatType_420YpCbCr8PlanarFullRange: pixelFormatName = @"kCVPixelFormatType_420YpCbCr8PlanarFullRange"; break; - case kCVPixelFormatType_422YpCbCr_4A_8BiPlanar: pixelFormatName = @"kCVPixelFormatType_422YpCbCr_4A_8BiPlanar"; break; - case kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange: pixelFormatName = @"kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange"; break; - case kCVPixelFormatType_420YpCbCr8BiPlanarFullRange: pixelFormatName = @"kCVPixelFormatType_420YpCbCr8BiPlanarFullRange"; break; - case kCVPixelFormatType_422YpCbCr8_yuvs: pixelFormatName = @"kCVPixelFormatType_422YpCbCr8_yuvs"; break; - case kCVPixelFormatType_422YpCbCr8FullRange: pixelFormatName = @"kCVPixelFormatType_422YpCbCr8FullRange"; break; - case kCVPixelFormatType_OneComponent8: pixelFormatName = @"kCVPixelFormatType_OneComponent8"; break; - case kCVPixelFormatType_TwoComponent8: pixelFormatName = @"kCVPixelFormatType_TwoComponent8"; break; - } - NSLog(@"Supported pixel format: %@", pixelFormatName); - } -} - -@end diff --git a/ScannerApp/Frameworks/GPUImage/framework/Source/Mac/GPUImageContext.h b/ScannerApp/Frameworks/GPUImage/framework/Source/Mac/GPUImageContext.h deleted file mode 100755 index d599601..0000000 --- a/ScannerApp/Frameworks/GPUImage/framework/Source/Mac/GPUImageContext.h +++ /dev/null @@ -1,55 +0,0 @@ -#import -#import -#import -#import "GLProgram.h" -#import "GPUImageFramebuffer.h" -#import "GPUImageFramebufferCache.h" - -#define GPUImageRotationSwapsWidthAndHeight(rotation) (((rotation) == kGPUImageRotateLeft) || ((rotation) == kGPUImageRotateRight) || ((rotation) == kGPUImageRotateRightFlipVertical) ) - -typedef enum { kGPUImageNoRotation, kGPUImageRotateLeft, kGPUImageRotateRight, kGPUImageFlipVertical, kGPUImageFlipHorizonal, kGPUImageRotateRightFlipVertical, kGPUImageRotateRightFlipHorizontal, kGPUImageRotate180 } GPUImageRotationMode; - -@interface GPUImageContext : NSObject - -@property(readonly, nonatomic) dispatch_queue_t contextQueue; -@property(readwrite, retain, nonatomic) GLProgram *currentShaderProgram; -@property(readonly, retain, nonatomic) NSOpenGLContext *context; -@property(readonly) CVOpenGLTextureCacheRef coreVideoTextureCache; -@property(readonly) GPUImageFramebufferCache *framebufferCache; - -+ (void *)contextKey; -+ (GPUImageContext *)sharedImageProcessingContext; -+ (dispatch_queue_t)sharedContextQueue; -+ (GPUImageFramebufferCache *)sharedFramebufferCache; -+ (void)useImageProcessingContext; -+ (void)setActiveShaderProgram:(GLProgram *)shaderProgram; -+ (GLint)maximumTextureSizeForThisDevice; -+ (GLint)maximumTextureUnitsForThisDevice; -+ (BOOL)deviceSupportsOpenGLESExtension:(NSString *)extension; -+ (BOOL)deviceSupportsRedTextures; -+ (BOOL)deviceSupportsFramebufferReads; -+ (CGSize)sizeThatFitsWithinATextureForSize:(CGSize)inputSize; - -- (void)presentBufferForDisplay; -- (GLProgram *)programForVertexShaderString:(NSString *)vertexShaderString fragmentShaderString:(NSString *)fragmentShaderString; - -- (void)useSharegroup:(CGLShareGroupObj *)sharegroup; - -// Manage fast texture upload -+ (BOOL)supportsFastTextureUpload; - -@end - -@protocol GPUImageInput -- (void)newFrameReadyAtTime:(CMTime)frameTime atIndex:(NSInteger)textureIndex; -- (void)setInputFramebuffer:(GPUImageFramebuffer *)newInputFramebuffer atIndex:(NSInteger)textureIndex; -- (NSInteger)nextAvailableTextureIndex; -- (void)setInputSize:(CGSize)newSize atIndex:(NSInteger)textureIndex; -- (void)setInputRotation:(GPUImageRotationMode)newInputRotation atIndex:(NSInteger)textureIndex; -- (CGSize)maximumOutputSize; -- (void)endProcessing; -- (BOOL)shouldIgnoreUpdatesToThisTarget; -- (BOOL)enabled; -- (BOOL)wantsMonochromeInput; -- (void)setCurrentlyReceivingMonochromeInput:(BOOL)newValue; -@end diff --git a/ScannerApp/Frameworks/GPUImage/framework/Source/Mac/GPUImageContext.m b/ScannerApp/Frameworks/GPUImage/framework/Source/Mac/GPUImageContext.m deleted file mode 100755 index 0532d95..0000000 --- a/ScannerApp/Frameworks/GPUImage/framework/Source/Mac/GPUImageContext.m +++ /dev/null @@ -1,268 +0,0 @@ -#import "GPUImageContext.h" -#import - -@interface GPUImageContext() -{ - NSMutableDictionary *shaderProgramCache; - CGLShareGroupObj *_sharegroup; - NSOpenGLPixelFormat *_pixelFormat; -} - -@end - -@implementation GPUImageContext - -@synthesize context = _context; -@synthesize currentShaderProgram = _currentShaderProgram; -@synthesize contextQueue = _contextQueue; -@synthesize coreVideoTextureCache = _coreVideoTextureCache; -@synthesize framebufferCache = _framebufferCache; - -static void *openGLESContextQueueKey; - -- (id)init; -{ - if (!(self = [super init])) - { - return nil; - } - - openGLESContextQueueKey = &openGLESContextQueueKey; - _contextQueue = dispatch_queue_create("com.sunsetlakesoftware.GPUImage.openGLESContextQueue", NULL); - dispatch_queue_set_specific(_contextQueue, openGLESContextQueueKey, (__bridge void *)self, NULL); - shaderProgramCache = [[NSMutableDictionary alloc] init]; - - return self; -} - -+ (void *)contextKey { - return openGLESContextQueueKey; -} - -// Based on Colin Wheeler's example here: http://cocoasamurai.blogspot.com/2011/04/singletons-your-doing-them-wrong.html -+ (GPUImageContext *)sharedImageProcessingContext; -{ - static dispatch_once_t pred; - static GPUImageContext *sharedImageProcessingContext = nil; - - dispatch_once(&pred, ^{ - sharedImageProcessingContext = [[[self class] alloc] init]; - }); - return sharedImageProcessingContext; -} - -+ (dispatch_queue_t)sharedContextQueue; -{ - return [[self sharedImageProcessingContext] contextQueue]; -} - -+ (GPUImageFramebufferCache *)sharedFramebufferCache; -{ - return [[self sharedImageProcessingContext] framebufferCache]; -} - -+ (void)useImageProcessingContext; -{ - NSOpenGLContext *imageProcessingContext = [[GPUImageContext sharedImageProcessingContext] context]; - if ([NSOpenGLContext currentContext] != imageProcessingContext) - { - [imageProcessingContext makeCurrentContext]; - } -} - -+ (void)setActiveShaderProgram:(GLProgram *)shaderProgram; -{ - GPUImageContext *sharedContext = [GPUImageContext sharedImageProcessingContext]; - NSOpenGLContext *imageProcessingContext = [sharedContext context]; - if ([NSOpenGLContext currentContext] != imageProcessingContext) - { - [imageProcessingContext makeCurrentContext]; - } - - if (sharedContext.currentShaderProgram != shaderProgram) - { - sharedContext.currentShaderProgram = shaderProgram; - [shaderProgram use]; - } -} - -+ (GLint)maximumTextureSizeForThisDevice; -{ - static dispatch_once_t pred; - static GLint maxTextureSize = 0; - - dispatch_once(&pred, ^{ - [self useImageProcessingContext]; - glGetIntegerv(GL_MAX_TEXTURE_SIZE, &maxTextureSize); - }); - - return maxTextureSize; -} - -+ (GLint)maximumTextureUnitsForThisDevice; -{ - GLint maxTextureUnits; - glGetIntegerv(GL_MAX_TEXTURE_IMAGE_UNITS, &maxTextureUnits); - return maxTextureUnits; -} - -+ (BOOL)deviceSupportsOpenGLESExtension:(NSString *)extension; -{ - static dispatch_once_t pred; - static NSArray *extensionNames = nil; - - // Cache extensions for later quick reference, since this won't change for a given device - dispatch_once(&pred, ^{ - [GPUImageContext useImageProcessingContext]; - NSString *extensionsString = [NSString stringWithCString:(const char *)glGetString(GL_EXTENSIONS) encoding:NSASCIIStringEncoding]; - extensionNames = [extensionsString componentsSeparatedByString:@" "]; - }); - - return [extensionNames containsObject:extension]; -} - -+ (BOOL)deviceSupportsFramebufferReads; -{ - return NO; -} - -// http://www.khronos.org/registry/gles/extensions/EXT/EXT_texture_rg.txt - -+ (BOOL)deviceSupportsRedTextures; -{ - static dispatch_once_t pred; - static BOOL supportsRedTextures = NO; - - dispatch_once(&pred, ^{ - supportsRedTextures = [GPUImageContext deviceSupportsOpenGLESExtension:@"GL_EXT_texture_rg"]; - }); - - return supportsRedTextures; -} - - -+ (CGSize)sizeThatFitsWithinATextureForSize:(CGSize)inputSize; -{ - GLint maxTextureSize = [self maximumTextureSizeForThisDevice]; - if ( (inputSize.width < maxTextureSize) && (inputSize.height < maxTextureSize) ) - { - return inputSize; - } - - CGSize adjustedSize; - if (inputSize.width > inputSize.height) - { - adjustedSize.width = (CGFloat)maxTextureSize; - adjustedSize.height = ((CGFloat)maxTextureSize / inputSize.width) * inputSize.height; - } - else - { - adjustedSize.height = (CGFloat)maxTextureSize; - adjustedSize.width = ((CGFloat)maxTextureSize / inputSize.height) * inputSize.width; - } - - return adjustedSize; -} - -- (void)presentBufferForDisplay; -{ - [self.context flushBuffer]; -} - -- (GLProgram *)programForVertexShaderString:(NSString *)vertexShaderString fragmentShaderString:(NSString *)fragmentShaderString; -{ - NSString *lookupKeyForShaderProgram = [NSString stringWithFormat:@"V: %@ - F: %@", vertexShaderString, fragmentShaderString]; - GLProgram *programFromCache = [shaderProgramCache objectForKey:lookupKeyForShaderProgram]; - - if (programFromCache == nil) - { - programFromCache = [[GLProgram alloc] initWithVertexShaderString:vertexShaderString fragmentShaderString:fragmentShaderString]; - [shaderProgramCache setObject:programFromCache forKey:lookupKeyForShaderProgram]; - } - - return programFromCache; -} - -- (void)useSharegroup:(CGLShareGroupObj *)sharegroup; -{ - NSAssert(_context == nil, @"Unable to use a share group when the context has already been created. Call this method before you use the context for the first time."); - - _sharegroup = sharegroup; -} - -- (NSOpenGLContext *)createContext; -{ - NSOpenGLPixelFormatAttribute pixelFormatAttributes[] = { - NSOpenGLPFADoubleBuffer, - NSOpenGLPFAAccelerated, 0, - 0 - }; - - _pixelFormat = [[NSOpenGLPixelFormat alloc] initWithAttributes:pixelFormatAttributes]; - if (_pixelFormat == nil) - { - NSLog(@"Error: No appropriate pixel format found"); - } - // TODO: Take into account the sharegroup - NSOpenGLContext *context = [[NSOpenGLContext alloc] initWithFormat:_pixelFormat shareContext:nil]; - - NSAssert(context != nil, @"Unable to create an OpenGL context. The GPUImage framework requires OpenGL support to work."); - return context; -} - -- (CVOpenGLTextureCacheRef)coreVideoTextureCache; -{ - if (_coreVideoTextureCache == NULL) - { - - CVReturn err = CVOpenGLTextureCacheCreate(kCFAllocatorDefault, NULL, (__bridge void *)[self context], [_pixelFormat CGLPixelFormatObj], NULL, &_coreVideoTextureCache); - - if (err) - { - NSAssert(NO, @"Error at CVOpenGLESTextureCacheCreate %d", err); - } - - } - - return _coreVideoTextureCache; -} - - -#pragma mark - -#pragma mark Manage fast texture upload - -+ (BOOL)supportsFastTextureUpload; -{ - // This may need to be redone to account for the Mac's accelerated data transfer methods - return NO; -} - -#pragma mark - -#pragma mark Accessors - -- (NSOpenGLContext *)context; -{ - if (_context == nil) - { - _context = [self createContext]; - [_context makeCurrentContext]; - - // Set up a few global settings for the image processing pipeline - glDisable(GL_DEPTH_TEST); - glEnable(GL_TEXTURE_2D); - } - - return _context; -} - -- (GPUImageFramebufferCache *)framebufferCache; -{ - if (_framebufferCache == nil) - { - _framebufferCache = [[GPUImageFramebufferCache alloc] init]; - } - - return _framebufferCache; -} - -@end diff --git a/ScannerApp/Frameworks/GPUImage/framework/Source/Mac/GPUImageMac-Info.plist b/ScannerApp/Frameworks/GPUImage/framework/Source/Mac/GPUImageMac-Info.plist deleted file mode 100644 index 88daf2c..0000000 --- a/ScannerApp/Frameworks/GPUImage/framework/Source/Mac/GPUImageMac-Info.plist +++ /dev/null @@ -1,30 +0,0 @@ - - - - - CFBundleDevelopmentRegion - English - CFBundleExecutable - ${EXECUTABLE_NAME} - CFBundleIconFile - - CFBundleIdentifier - com.sunsetlakesoftware.${PRODUCT_NAME:rfc1034identifier} - CFBundleInfoDictionaryVersion - 6.0 - CFBundleName - ${PRODUCT_NAME} - CFBundlePackageType - FMWK - CFBundleShortVersionString - 1.0 - CFBundleSignature - ???? - CFBundleVersion - 1 - NSHumanReadableCopyright - Copyright © 2013 Sunset Lake Software LLC. All rights reserved. - NSPrincipalClass - - - diff --git a/ScannerApp/Frameworks/GPUImage/framework/Source/Mac/GPUImageMac-Prefix.pch b/ScannerApp/Frameworks/GPUImage/framework/Source/Mac/GPUImageMac-Prefix.pch deleted file mode 100644 index f01e581..0000000 --- a/ScannerApp/Frameworks/GPUImage/framework/Source/Mac/GPUImageMac-Prefix.pch +++ /dev/null @@ -1,7 +0,0 @@ -// -// Prefix header for all source files of the 'GPUImageMac' target in the 'GPUImageMac' project -// - -#ifdef __OBJC__ - #import -#endif diff --git a/ScannerApp/Frameworks/GPUImage/framework/Source/Mac/GPUImageMovieWriter.h b/ScannerApp/Frameworks/GPUImage/framework/Source/Mac/GPUImageMovieWriter.h deleted file mode 100755 index 530e5a6..0000000 --- a/ScannerApp/Frameworks/GPUImage/framework/Source/Mac/GPUImageMovieWriter.h +++ /dev/null @@ -1,57 +0,0 @@ -#import -#import -#import "GPUImageContext.h" - -extern NSString *const kGPUImageColorSwizzlingFragmentShaderString; - -@protocol GPUImageMovieWriterDelegate - -@optional -- (void)movieRecordingCompleted; -- (void)movieRecordingFailedWithError:(NSError*)error; - -@end - -@interface GPUImageMovieWriter : NSObject -{ - CMVideoDimensions videoDimensions; - CMVideoCodecType videoType; - - NSURL *movieURL; - NSString *fileType; - AVAssetWriter *assetWriter; - AVAssetWriterInput *assetWriterAudioInput; - AVAssetWriterInput *assetWriterVideoInput; - AVAssetWriterInputPixelBufferAdaptor *assetWriterPixelBufferInput; - dispatch_queue_t movieWritingQueue; - - CGSize videoSize; - GPUImageRotationMode inputRotation; -} - -@property(readwrite, nonatomic) BOOL hasAudioTrack; -@property(readwrite, nonatomic) BOOL shouldPassthroughAudio; -@property(nonatomic, copy) void(^completionBlock)(void); -@property(nonatomic, copy) void(^failureBlock)(NSError*); -@property(nonatomic, assign) id delegate; -@property(readwrite, nonatomic) BOOL encodingLiveVideo; -@property(nonatomic, copy) void(^videoInputReadyCallback)(void); -@property(nonatomic, copy) void(^audioInputReadyCallback)(void); -@property(nonatomic) BOOL enabled; - -// Initialization and teardown -- (id)initWithMovieURL:(NSURL *)newMovieURL size:(CGSize)newSize; -- (id)initWithMovieURL:(NSURL *)newMovieURL size:(CGSize)newSize fileType:(NSString *)newFileType outputSettings:(NSMutableDictionary *)outputSettings; - -- (void)setHasAudioTrack:(BOOL)hasAudioTrack audioSettings:(NSDictionary *)audioOutputSettings; - -// Movie recording -- (void)startRecording; -- (void)startRecordingInOrientation:(CGAffineTransform)orientationTransform; -- (void)finishRecording; -- (void)finishRecordingWithCompletionHandler:(void (^)(void))handler; -- (void)cancelRecording; -- (void)processAudioBuffer:(CMSampleBufferRef)audioBuffer; -- (void)enableSynchronizationCallbacks; - -@end diff --git a/ScannerApp/Frameworks/GPUImage/framework/Source/Mac/GPUImageMovieWriter.m b/ScannerApp/Frameworks/GPUImage/framework/Source/Mac/GPUImageMovieWriter.m deleted file mode 100755 index 89f0a37..0000000 --- a/ScannerApp/Frameworks/GPUImage/framework/Source/Mac/GPUImageMovieWriter.m +++ /dev/null @@ -1,644 +0,0 @@ -#import "GPUImageMovieWriter.h" - -#import "GPUImageContext.h" -#import "GLProgram.h" -#import "GPUImageFilter.h" - -#if TARGET_IPHONE_SIMULATOR || TARGET_OS_IPHONE -NSString *const kGPUImageColorSwizzlingFragmentShaderString = SHADER_STRING -( - varying highp vec2 textureCoordinate; - - uniform sampler2D inputImageTexture; - - void main() - { - gl_FragColor = texture2D(inputImageTexture, textureCoordinate).bgra; - } - ); -#else -NSString *const kGPUImageColorSwizzlingFragmentShaderString = SHADER_STRING -( - varying vec2 textureCoordinate; - - uniform sampler2D inputImageTexture; - - void main() - { - gl_FragColor = texture2D(inputImageTexture, textureCoordinate).bgra; - } - ); -#endif - - -@interface GPUImageMovieWriter () -{ - GPUImageFramebuffer *firstInputFramebuffer; - - GLuint movieFramebuffer, movieRenderbuffer; - - GLProgram *colorSwizzlingProgram; - GLint colorSwizzlingPositionAttribute, colorSwizzlingTextureCoordinateAttribute; - GLint colorSwizzlingInputTextureUniform; - - GLubyte *frameData; - - CMTime startTime, previousFrameTime; - - BOOL isRecording; -} - -// Movie recording -- (void)initializeMovieWithOutputSettings:(NSMutableDictionary *)outputSettings; - -// Frame rendering -- (void)createDataFBO; -- (void)destroyDataFBO; -- (void)setFilterFBO; - -- (void)renderAtInternalSize; - -@end - -@implementation GPUImageMovieWriter - -@synthesize hasAudioTrack = _hasAudioTrack; -@synthesize encodingLiveVideo = _encodingLiveVideo; -@synthesize shouldPassthroughAudio = _shouldPassthroughAudio; -@synthesize completionBlock; -@synthesize failureBlock; -@synthesize videoInputReadyCallback; -@synthesize audioInputReadyCallback; -@synthesize enabled; - -@synthesize delegate = _delegate; - -#pragma mark - -#pragma mark Initialization and teardown - -- (id)initWithMovieURL:(NSURL *)newMovieURL size:(CGSize)newSize; -{ - return [self initWithMovieURL:newMovieURL size:newSize fileType:AVFileTypeQuickTimeMovie outputSettings:nil]; -} - -- (id)initWithMovieURL:(NSURL *)newMovieURL size:(CGSize)newSize fileType:(NSString *)newFileType outputSettings:(NSMutableDictionary *)outputSettings; -{ - if (!(self = [super init])) - { - return nil; - } - - self.enabled = YES; - - videoSize = newSize; - movieURL = newMovieURL; - fileType = newFileType; - startTime = kCMTimeInvalid; - _encodingLiveVideo = YES; - previousFrameTime = kCMTimeNegativeInfinity; - inputRotation = kGPUImageNoRotation; - - runSynchronouslyOnVideoProcessingQueue(^{ - [GPUImageContext useImageProcessingContext]; - - if ([GPUImageContext supportsFastTextureUpload]) - { - colorSwizzlingProgram = [[GPUImageContext sharedImageProcessingContext] programForVertexShaderString:kGPUImageVertexShaderString fragmentShaderString:kGPUImagePassthroughFragmentShaderString]; - } - else - { - colorSwizzlingProgram = [[GPUImageContext sharedImageProcessingContext] programForVertexShaderString:kGPUImageVertexShaderString fragmentShaderString:kGPUImageColorSwizzlingFragmentShaderString]; - } - - if (!colorSwizzlingProgram.initialized) - { - [colorSwizzlingProgram addAttribute:@"position"]; - [colorSwizzlingProgram addAttribute:@"inputTextureCoordinate"]; - - if (![colorSwizzlingProgram link]) - { - NSString *progLog = [colorSwizzlingProgram programLog]; - NSLog(@"Program link log: %@", progLog); - NSString *fragLog = [colorSwizzlingProgram fragmentShaderLog]; - NSLog(@"Fragment shader compile log: %@", fragLog); - NSString *vertLog = [colorSwizzlingProgram vertexShaderLog]; - NSLog(@"Vertex shader compile log: %@", vertLog); - colorSwizzlingProgram = nil; - NSAssert(NO, @"Filter shader link failed"); - } - } - - colorSwizzlingPositionAttribute = [colorSwizzlingProgram attributeIndex:@"position"]; - colorSwizzlingTextureCoordinateAttribute = [colorSwizzlingProgram attributeIndex:@"inputTextureCoordinate"]; - colorSwizzlingInputTextureUniform = [colorSwizzlingProgram uniformIndex:@"inputImageTexture"]; - - // REFACTOR: Wrap this in a block for the image processing queue - [GPUImageContext setActiveShaderProgram:colorSwizzlingProgram]; - - glEnableVertexAttribArray(colorSwizzlingPositionAttribute); - glEnableVertexAttribArray(colorSwizzlingTextureCoordinateAttribute); - }); - - [self initializeMovieWithOutputSettings:outputSettings]; - - return self; -} - -- (void)dealloc; -{ - [self destroyDataFBO]; - - if (frameData != NULL) - { - free(frameData); - } -} - -#pragma mark - -#pragma mark Movie recording - -- (void)initializeMovieWithOutputSettings:(NSMutableDictionary *)outputSettings; -{ - isRecording = NO; - - self.enabled = YES; - frameData = (GLubyte *) malloc((int)videoSize.width * (int)videoSize.height * 4); - -// frameData = (GLubyte *) calloc(videoSize.width * videoSize.height * 4, sizeof(GLubyte)); - NSError *error = nil; - assetWriter = [[AVAssetWriter alloc] initWithURL:movieURL fileType:fileType error:&error]; - if (error != nil) - { - NSLog(@"Error: %@", error); - if (failureBlock) - { - failureBlock(error); - } - else - { - if(self.delegate && [self.delegate respondsToSelector:@selector(movieRecordingFailedWithError:)]) - { - [self.delegate movieRecordingFailedWithError:error]; - } - } - } - - // Set this to make sure that a functional movie is produced, even if the recording is cut off mid-stream. Only the last second should be lost in that case. - assetWriter.movieFragmentInterval = CMTimeMakeWithSeconds(1.0, 1000); - - // use default output settings if none specified - if (outputSettings == nil) - { - outputSettings = [[NSMutableDictionary alloc] init]; - [outputSettings setObject:AVVideoCodecH264 forKey:AVVideoCodecKey]; - [outputSettings setObject:[NSNumber numberWithInt:videoSize.width] forKey:AVVideoWidthKey]; - [outputSettings setObject:[NSNumber numberWithInt:videoSize.height] forKey:AVVideoHeightKey]; - } - // custom output settings specified - else - { - #ifndef NS_BLOCK_ASSERTIONS - NSString *videoCodec = [outputSettings objectForKey:AVVideoCodecKey]; - NSNumber *width = [outputSettings objectForKey:AVVideoWidthKey]; - NSNumber *height = [outputSettings objectForKey:AVVideoHeightKey]; - - NSAssert(videoCodec && width && height, @"OutputSettings is missing required parameters."); - #endif - } - - /* - NSDictionary *videoCleanApertureSettings = [NSDictionary dictionaryWithObjectsAndKeys: - [NSNumber numberWithInt:videoSize.width], AVVideoCleanApertureWidthKey, - [NSNumber numberWithInt:videoSize.height], AVVideoCleanApertureHeightKey, - [NSNumber numberWithInt:0], AVVideoCleanApertureHorizontalOffsetKey, - [NSNumber numberWithInt:0], AVVideoCleanApertureVerticalOffsetKey, - nil]; - - NSDictionary *videoAspectRatioSettings = [NSDictionary dictionaryWithObjectsAndKeys: - [NSNumber numberWithInt:3], AVVideoPixelAspectRatioHorizontalSpacingKey, - [NSNumber numberWithInt:3], AVVideoPixelAspectRatioVerticalSpacingKey, - nil]; - - NSMutableDictionary * compressionProperties = [[NSMutableDictionary alloc] init]; - [compressionProperties setObject:videoCleanApertureSettings forKey:AVVideoCleanApertureKey]; - [compressionProperties setObject:videoAspectRatioSettings forKey:AVVideoPixelAspectRatioKey]; - [compressionProperties setObject:[NSNumber numberWithInt: 2000000] forKey:AVVideoAverageBitRateKey]; - [compressionProperties setObject:[NSNumber numberWithInt: 16] forKey:AVVideoMaxKeyFrameIntervalKey]; - [compressionProperties setObject:AVVideoProfileLevelH264Main31 forKey:AVVideoProfileLevelKey]; - - [outputSettings setObject:compressionProperties forKey:AVVideoCompressionPropertiesKey]; - */ - - assetWriterVideoInput = [AVAssetWriterInput assetWriterInputWithMediaType:AVMediaTypeVideo outputSettings:outputSettings]; - assetWriterVideoInput.expectsMediaDataInRealTime = _encodingLiveVideo; - - // You need to use BGRA for the video in order to get realtime encoding. I use a color-swizzling shader to line up glReadPixels' normal RGBA output with the movie input's BGRA. - NSDictionary *sourcePixelBufferAttributesDictionary = [NSDictionary dictionaryWithObjectsAndKeys: [NSNumber numberWithInt:kCVPixelFormatType_32BGRA], kCVPixelBufferPixelFormatTypeKey, - [NSNumber numberWithInt:videoSize.width], kCVPixelBufferWidthKey, - [NSNumber numberWithInt:videoSize.height], kCVPixelBufferHeightKey, - nil]; -// NSDictionary *sourcePixelBufferAttributesDictionary = [NSDictionary dictionaryWithObjectsAndKeys: [NSNumber numberWithInt:kCVPixelFormatType_32ARGB], kCVPixelBufferPixelFormatTypeKey, -// nil]; - - assetWriterPixelBufferInput = [AVAssetWriterInputPixelBufferAdaptor assetWriterInputPixelBufferAdaptorWithAssetWriterInput:assetWriterVideoInput sourcePixelBufferAttributes:sourcePixelBufferAttributesDictionary]; - - [assetWriter addInput:assetWriterVideoInput]; -} - -- (void)startRecording; -{ - isRecording = YES; - startTime = kCMTimeInvalid; - // [assetWriter startWriting]; - - // [assetWriter startSessionAtSourceTime:kCMTimeZero]; -} - -- (void)startRecordingInOrientation:(CGAffineTransform)orientationTransform; -{ - assetWriterVideoInput.transform = orientationTransform; - - [self startRecording]; -} - -- (void)cancelRecording; -{ - if (assetWriter.status == AVAssetWriterStatusCompleted) - { - return; - } - - isRecording = NO; - runOnMainQueueWithoutDeadlocking(^{ - [assetWriterVideoInput markAsFinished]; - [assetWriterAudioInput markAsFinished]; - [assetWriter cancelWriting]; - }); -} - -- (void)finishRecording; -{ - [self finishRecordingWithCompletionHandler:nil]; -} - -- (void)finishRecordingWithCompletionHandler:(void (^)(void))handler; -{ - if (assetWriter.status == AVAssetWriterStatusCompleted) - { - return; - } - - isRecording = NO; - runOnMainQueueWithoutDeadlocking(^{ - [assetWriterVideoInput markAsFinished]; - [assetWriterAudioInput markAsFinished]; -#if (!defined(__IPHONE_6_0) || (__IPHONE_OS_VERSION_MAX_ALLOWED < __IPHONE_6_0)) - // Not iOS 6 SDK - [assetWriter finishWriting]; - if (handler) handler(); -#else - // iOS 6 SDK - if ([assetWriter respondsToSelector:@selector(finishWritingWithCompletionHandler:)]) { - // Running iOS 6 - [assetWriter finishWritingWithCompletionHandler:(handler ?: ^{ })]; - } - else { - // Not running iOS 6 - [assetWriter finishWriting]; - if (handler) handler(); - } -#endif - }); -} - -- (void)processAudioBuffer:(CMSampleBufferRef)audioBuffer; -{ - if (!isRecording) - { - return; - } - - if (_hasAudioTrack) - { - CMTime currentSampleTime = CMSampleBufferGetOutputPresentationTimeStamp(audioBuffer); - - if (CMTIME_IS_INVALID(startTime)) - { - if (audioInputReadyCallback == NULL) - { - [assetWriter startWriting]; - } - [assetWriter startSessionAtSourceTime:currentSampleTime]; - startTime = currentSampleTime; - } - - if (!assetWriterAudioInput.readyForMoreMediaData) - { - NSLog(@"Had to drop an audio frame"); - return; - } - -// NSLog(@"Recorded audio sample time: %lld, %d, %lld", currentSampleTime.value, currentSampleTime.timescale, currentSampleTime.epoch); - [assetWriterAudioInput appendSampleBuffer:audioBuffer]; - } -} - -- (void)enableSynchronizationCallbacks; -{ - if (videoInputReadyCallback != NULL) - { - [assetWriter startWriting]; - [assetWriterVideoInput requestMediaDataWhenReadyOnQueue:[GPUImageContext sharedContextQueue] usingBlock:videoInputReadyCallback]; - } - - if (audioInputReadyCallback != NULL) - { - [assetWriterAudioInput requestMediaDataWhenReadyOnQueue:[GPUImageContext sharedContextQueue] usingBlock:audioInputReadyCallback]; - } - -} - -#pragma mark - -#pragma mark Frame rendering - -- (void)createDataFBO; -{ - glActiveTexture(GL_TEXTURE1); - glGenFramebuffers(1, &movieFramebuffer); - glBindFramebuffer(GL_FRAMEBUFFER, movieFramebuffer); - - glGenRenderbuffers(1, &movieRenderbuffer); - glBindRenderbuffer(GL_RENDERBUFFER, movieRenderbuffer); - glRenderbufferStorage(GL_RENDERBUFFER, GL_RGBA8, (int)videoSize.width, (int)videoSize.height); - glFramebufferRenderbuffer(GL_FRAMEBUFFER, GL_COLOR_ATTACHMENT0, GL_RENDERBUFFER, movieRenderbuffer); - - #ifndef NS_BLOCK_ASSERTIONS - GLenum status = glCheckFramebufferStatus(GL_FRAMEBUFFER); - NSAssert(status == GL_FRAMEBUFFER_COMPLETE, @"Incomplete filter FBO: %d", status); - #endif -} - -- (void)destroyDataFBO; -{ - [GPUImageContext useImageProcessingContext]; - - if (movieFramebuffer) - { - glDeleteFramebuffers(1, &movieFramebuffer); - movieFramebuffer = 0; - } - - if (movieRenderbuffer) - { - glDeleteRenderbuffers(1, &movieRenderbuffer); - movieRenderbuffer = 0; - } -} - -- (void)setFilterFBO; -{ - if (!movieFramebuffer) - { - [self createDataFBO]; - } - - glBindFramebuffer(GL_FRAMEBUFFER, movieFramebuffer); - - glViewport(0, 0, (int)videoSize.width, (int)videoSize.height); -} - -- (void)renderAtInternalSize; -{ - [GPUImageContext useImageProcessingContext]; - [self setFilterFBO]; - - [GPUImageContext setActiveShaderProgram:colorSwizzlingProgram]; - - glClearColor(1.0f, 0.0f, 0.0f, 1.0f); - glClear(GL_COLOR_BUFFER_BIT | GL_DEPTH_BUFFER_BIT); - - // This needs to be flipped to write out to video correctly - static const GLfloat squareVertices[] = { - -1.0f, -1.0f, - 1.0f, -1.0f, - -1.0f, 1.0f, - 1.0f, 1.0f, - }; - - static const GLfloat textureCoordinates[] = { - 0.0f, 0.0f, - 1.0f, 0.0f, - 0.0f, 1.0f, - 1.0f, 1.0f, - }; - - glActiveTexture(GL_TEXTURE4); - glBindTexture(GL_TEXTURE_2D, [firstInputFramebuffer texture]); - glUniform1i(colorSwizzlingInputTextureUniform, 4); - - glVertexAttribPointer(colorSwizzlingPositionAttribute, 2, GL_FLOAT, 0, 0, squareVertices); - glVertexAttribPointer(colorSwizzlingTextureCoordinateAttribute, 2, GL_FLOAT, 0, 0, textureCoordinates); - - glDrawArrays(GL_TRIANGLE_STRIP, 0, 4); - - glFinish(); - [firstInputFramebuffer unlock]; -} - -#pragma mark - -#pragma mark GPUImageInput protocol - -- (void)newFrameReadyAtTime:(CMTime)frameTime atIndex:(NSInteger)textureIndex; -{ - if (!isRecording) - { - [firstInputFramebuffer unlock]; - return; - } - - // Drop frames forced by images and other things with no time constants - // Also, if two consecutive times with the same value are added to the movie, it aborts recording, so I bail on that case - if ( (CMTIME_IS_INVALID(frameTime)) || (CMTIME_COMPARE_INLINE(frameTime, ==, previousFrameTime)) || (CMTIME_IS_INDEFINITE(frameTime)) ) - { - [firstInputFramebuffer unlock]; - return; - } - - if (CMTIME_IS_INVALID(startTime)) - { - if (videoInputReadyCallback == NULL) - { - [assetWriter startWriting]; - } - - [assetWriter startSessionAtSourceTime:frameTime]; - startTime = frameTime; - } - - if (!assetWriterVideoInput.readyForMoreMediaData) - { - [firstInputFramebuffer unlock]; - NSLog(@"Had to drop a video frame"); - return; - } - - // Render the frame with swizzled colors, so that they can be uploaded quickly as BGRA frames - [GPUImageContext useImageProcessingContext]; - [self renderAtInternalSize]; - - CVPixelBufferRef pixel_buffer = NULL; - - CVReturn status = CVPixelBufferPoolCreatePixelBuffer (NULL, [assetWriterPixelBufferInput pixelBufferPool], &pixel_buffer); - if ((pixel_buffer == NULL) || (status != kCVReturnSuccess)) - { - return; - } - else - { - CVPixelBufferLockBaseAddress(pixel_buffer, 0); - - GLubyte *pixelBufferData = (GLubyte *)CVPixelBufferGetBaseAddress(pixel_buffer); - glReadPixels(0, 0, videoSize.width, videoSize.height, GL_RGBA, GL_UNSIGNED_BYTE, pixelBufferData); - } - -// if(![assetWriterPixelBufferInput appendPixelBuffer:pixel_buffer withPresentationTime:CMTimeSubtract(frameTime, startTime)]) - if(![assetWriterPixelBufferInput appendPixelBuffer:pixel_buffer withPresentationTime:frameTime]) - { - NSLog(@"Problem appending pixel buffer at time: %lld", frameTime.value); - } - else - { -// NSLog(@"Recorded video sample time: %lld, %d, %lld", frameTime.value, frameTime.timescale, frameTime.epoch); - } - CVPixelBufferUnlockBaseAddress(pixel_buffer, 0); - - previousFrameTime = frameTime; - - if (![GPUImageContext supportsFastTextureUpload]) - { - CVPixelBufferRelease(pixel_buffer); - } -} - -- (NSInteger)nextAvailableTextureIndex; -{ - return 0; -} - -- (void)setInputFramebuffer:(GPUImageFramebuffer *)newInputFramebuffer atIndex:(NSInteger)textureIndex; -{ - firstInputFramebuffer = newInputFramebuffer; - [firstInputFramebuffer lock]; -} - -- (void)setInputRotation:(GPUImageRotationMode)newInputRotation atIndex:(NSInteger)textureIndex; -{ - inputRotation = newInputRotation; -} - -- (void)setInputSize:(CGSize)newSize atIndex:(NSInteger)textureIndex; -{ -} - -- (CGSize)maximumOutputSize; -{ - return videoSize; -} - -- (void)endProcessing -{ - if (completionBlock) - { - completionBlock(); - } - else - { - if (_delegate && [_delegate respondsToSelector:@selector(movieRecordingCompleted)]) - { - [_delegate movieRecordingCompleted]; - } - } -} - -- (BOOL)shouldIgnoreUpdatesToThisTarget; -{ - return NO; -} - -- (void)conserveMemoryForNextFrame; -{ - -} - -- (BOOL)wantsMonochromeInput; -{ - return NO; -} - -- (void)setCurrentlyReceivingMonochromeInput:(BOOL)newValue; -{ - -} - -#pragma mark - -#pragma mark Accessors - -- (void)setHasAudioTrack:(BOOL)newValue -{ - [self setHasAudioTrack:newValue audioSettings:nil]; -} - -- (void)setHasAudioTrack:(BOOL)newValue audioSettings:(NSDictionary *)audioOutputSettings; -{ - _hasAudioTrack = newValue; - - if (_hasAudioTrack) - { - if (_shouldPassthroughAudio) - { - // Do not set any settings so audio will be the same as passthrough - audioOutputSettings = nil; - } - else if (audioOutputSettings == nil) - { -// double preferredHardwareSampleRate = [[AVAudioSession sharedInstance] currentHardwareSampleRate]; - double preferredHardwareSampleRate = 48000; // ? - TODO: Fix this, because it's probably broken - - AudioChannelLayout acl; - bzero( &acl, sizeof(acl)); - acl.mChannelLayoutTag = kAudioChannelLayoutTag_Mono; - - audioOutputSettings = [NSDictionary dictionaryWithObjectsAndKeys: - [ NSNumber numberWithInt: kAudioFormatMPEG4AAC], AVFormatIDKey, - [ NSNumber numberWithInt: 1 ], AVNumberOfChannelsKey, - [ NSNumber numberWithFloat: preferredHardwareSampleRate ], AVSampleRateKey, - [ NSData dataWithBytes: &acl length: sizeof( acl ) ], AVChannelLayoutKey, - //[ NSNumber numberWithInt:AVAudioQualityLow], AVEncoderAudioQualityKey, - [ NSNumber numberWithInt: 64000 ], AVEncoderBitRateKey, - nil]; -/* - AudioChannelLayout acl; - bzero( &acl, sizeof(acl)); - acl.mChannelLayoutTag = kAudioChannelLayoutTag_Mono; - - audioOutputSettings = [NSDictionary dictionaryWithObjectsAndKeys: - [ NSNumber numberWithInt: kAudioFormatMPEG4AAC ], AVFormatIDKey, - [ NSNumber numberWithInt: 1 ], AVNumberOfChannelsKey, - [ NSNumber numberWithFloat: 44100.0 ], AVSampleRateKey, - [ NSNumber numberWithInt: 64000 ], AVEncoderBitRateKey, - [ NSData dataWithBytes: &acl length: sizeof( acl ) ], AVChannelLayoutKey, - nil];*/ - } - - assetWriterAudioInput = [AVAssetWriterInput assetWriterInputWithMediaType:AVMediaTypeAudio outputSettings:audioOutputSettings]; - [assetWriter addInput:assetWriterAudioInput]; - assetWriterAudioInput.expectsMediaDataInRealTime = _encodingLiveVideo; - } - else - { - // Remove audio track if it exists - } -} - - -@end diff --git a/ScannerApp/Frameworks/GPUImage/framework/Source/Mac/GPUImagePicture.h b/ScannerApp/Frameworks/GPUImage/framework/Source/Mac/GPUImagePicture.h deleted file mode 100755 index 6597053..0000000 --- a/ScannerApp/Frameworks/GPUImage/framework/Source/Mac/GPUImagePicture.h +++ /dev/null @@ -1,25 +0,0 @@ -#import -#import "GPUImageOutput.h" - -@interface GPUImagePicture : GPUImageOutput -{ - CGSize pixelSizeOfImage; - BOOL hasProcessedImage; - - dispatch_semaphore_t imageUpdateSemaphore; -} - -// Initialization and teardown -- (id)initWithURL:(NSURL *)url; -- (id)initWithImage:(NSImage *)newImageSource; -- (id)initWithCGImage:(CGImageRef)newImageSource; -- (id)initWithImage:(NSImage *)newImageSource smoothlyScaleOutput:(BOOL)smoothlyScaleOutput; -- (id)initWithCGImage:(CGImageRef)newImageSource smoothlyScaleOutput:(BOOL)smoothlyScaleOutput; - -// Image rendering -- (void)processImage; -- (BOOL)processImageWithCompletionHandler:(void (^)(void))completion; -- (void)processImageUpToFilter:(GPUImageOutput *)finalFilterInChain withCompletionHandler:(void (^)(NSImage *processedImage))block; -- (CGSize)outputImageSize; - -@end diff --git a/ScannerApp/Frameworks/GPUImage/framework/Source/Mac/GPUImagePicture.m b/ScannerApp/Frameworks/GPUImage/framework/Source/Mac/GPUImagePicture.m deleted file mode 100755 index 1ef37bf..0000000 --- a/ScannerApp/Frameworks/GPUImage/framework/Source/Mac/GPUImagePicture.m +++ /dev/null @@ -1,299 +0,0 @@ -#import "GPUImagePicture.h" - -@implementation GPUImagePicture - -#pragma mark - -#pragma mark Initialization and teardown - -- (id)initWithURL:(NSURL *)url; -{ - NSData *imageData = [[NSData alloc] initWithContentsOfURL:url]; - - if (!(self = [self initWithData:imageData])) - { - return nil; - } - - return self; -} - -- (id)initWithData:(NSData *)imageData; -{ - NSImage *inputImage = [[NSImage alloc] initWithData:imageData]; - - if (!(self = [self initWithImage:inputImage])) - { - return nil; - } - - return self; -} - -- (id)initWithImage:(NSImage *)newImageSource; -{ - if (!(self = [self initWithImage:newImageSource smoothlyScaleOutput:NO])) - { - return nil; - } - - return self; -} - -- (id)initWithCGImage:(CGImageRef)newImageSource; -{ - if (!(self = [self initWithCGImage:newImageSource smoothlyScaleOutput:NO])) - { - return nil; - } - return self; -} - -- (id)initWithImage:(NSImage *)newImageSource smoothlyScaleOutput:(BOOL)smoothlyScaleOutput; -{ - return [self initWithCGImage:[newImageSource CGImageForProposedRect:NULL context:NULL hints:nil] smoothlyScaleOutput:smoothlyScaleOutput]; -} - -- (id)initWithCGImage:(CGImageRef)newImageSource smoothlyScaleOutput:(BOOL)smoothlyScaleOutput; -{ - if (!(self = [super init])) - { - return nil; - } - - hasProcessedImage = NO; - self.shouldSmoothlyScaleOutput = smoothlyScaleOutput; - imageUpdateSemaphore = dispatch_semaphore_create(1); - - // TODO: Dispatch this whole thing asynchronously to move image loading off main thread - CGFloat widthOfImage = CGImageGetWidth(newImageSource); - CGFloat heightOfImage = CGImageGetHeight(newImageSource); - - // If passed an empty image reference, CGContextDrawImage will fail in future versions of the SDK. - NSAssert( widthOfImage > 0 && heightOfImage > 0, @"Passed image must not be empty - it should be at least 1px tall and wide"); - - pixelSizeOfImage = CGSizeMake(widthOfImage, heightOfImage); - CGSize pixelSizeToUseForTexture = pixelSizeOfImage; - - BOOL shouldRedrawUsingCoreGraphics = NO; - - // For now, deal with images larger than the maximum texture size by resizing to be within that limit - CGSize scaledImageSizeToFitOnGPU = [GPUImageContext sizeThatFitsWithinATextureForSize:pixelSizeOfImage]; - if (!CGSizeEqualToSize(scaledImageSizeToFitOnGPU, pixelSizeOfImage)) - { - pixelSizeOfImage = scaledImageSizeToFitOnGPU; - pixelSizeToUseForTexture = pixelSizeOfImage; - shouldRedrawUsingCoreGraphics = YES; - } - - if (self.shouldSmoothlyScaleOutput) - { - // In order to use mipmaps, you need to provide power-of-two textures, so convert to the next largest power of two and stretch to fill - CGFloat powerClosestToWidth = ceil(log2(pixelSizeOfImage.width)); - CGFloat powerClosestToHeight = ceil(log2(pixelSizeOfImage.height)); - - pixelSizeToUseForTexture = CGSizeMake(pow(2.0, powerClosestToWidth), pow(2.0, powerClosestToHeight)); - - shouldRedrawUsingCoreGraphics = YES; - } - - GLubyte *imageData = NULL; - CFDataRef dataFromImageDataProvider; - GLenum format = GL_BGRA; - - if (!shouldRedrawUsingCoreGraphics) { - /* Check that the memory layout is compatible with GL, as we cannot use glPixelStore to - * tell GL about the memory layout with GLES. - */ - if (CGImageGetBytesPerRow(newImageSource) != CGImageGetWidth(newImageSource) * 4 || - CGImageGetBitsPerPixel(newImageSource) != 32 || - CGImageGetBitsPerComponent(newImageSource) != 8) - { - shouldRedrawUsingCoreGraphics = YES; - } else { - /* Check that the bitmap pixel format is compatible with GL */ - CGBitmapInfo bitmapInfo = CGImageGetBitmapInfo(newImageSource); - if ((bitmapInfo & kCGBitmapFloatComponents) != 0) { - /* We don't support float components for use directly in GL */ - shouldRedrawUsingCoreGraphics = YES; - } else { - CGBitmapInfo byteOrderInfo = bitmapInfo & kCGBitmapByteOrderMask; - if (byteOrderInfo == kCGBitmapByteOrder32Little) { - /* Little endian, for alpha-first we can use this bitmap directly in GL */ - CGImageAlphaInfo alphaInfo = bitmapInfo & kCGBitmapAlphaInfoMask; - if (alphaInfo != kCGImageAlphaPremultipliedFirst && alphaInfo != kCGImageAlphaFirst && - alphaInfo != kCGImageAlphaNoneSkipFirst) { - shouldRedrawUsingCoreGraphics = YES; - } - } else if (byteOrderInfo == kCGBitmapByteOrderDefault || byteOrderInfo == kCGBitmapByteOrder32Big) { - /* Big endian, for alpha-last we can use this bitmap directly in GL */ - CGImageAlphaInfo alphaInfo = bitmapInfo & kCGBitmapAlphaInfoMask; - if (alphaInfo != kCGImageAlphaPremultipliedLast && alphaInfo != kCGImageAlphaLast && - alphaInfo != kCGImageAlphaNoneSkipLast) { - shouldRedrawUsingCoreGraphics = YES; - } else { - /* Can access directly using GL_RGBA pixel format */ - format = GL_RGBA; - } - } - } - } - } - - // CFAbsoluteTime elapsedTime, startTime = CFAbsoluteTimeGetCurrent(); - - if (shouldRedrawUsingCoreGraphics) - { - // For resized or incompatible image: redraw - imageData = (GLubyte *) calloc(1, (int)pixelSizeToUseForTexture.width * (int)pixelSizeToUseForTexture.height * 4); - - CGColorSpaceRef genericRGBColorspace = CGColorSpaceCreateDeviceRGB(); - - CGContextRef imageContext = CGBitmapContextCreate(imageData, (size_t)pixelSizeToUseForTexture.width, (size_t)pixelSizeToUseForTexture.height, 8, (size_t)pixelSizeToUseForTexture.width * 4, genericRGBColorspace, kCGBitmapByteOrder32Little | kCGImageAlphaPremultipliedFirst); - // CGContextSetBlendMode(imageContext, kCGBlendModeCopy); // From Technical Q&A QA1708: http://developer.apple.com/library/ios/#qa/qa1708/_index.html - CGContextDrawImage(imageContext, CGRectMake(0.0, 0.0, pixelSizeToUseForTexture.width, pixelSizeToUseForTexture.height), newImageSource); - CGContextRelease(imageContext); - CGColorSpaceRelease(genericRGBColorspace); - } - else - { - // Access the raw image bytes directly - dataFromImageDataProvider = CGDataProviderCopyData(CGImageGetDataProvider(newImageSource)); - imageData = (GLubyte *)CFDataGetBytePtr(dataFromImageDataProvider); - } - - // elapsedTime = (CFAbsoluteTimeGetCurrent() - startTime) * 1000.0; - // NSLog(@"Core Graphics drawing time: %f", elapsedTime); - - // CGFloat currentRedTotal = 0.0f, currentGreenTotal = 0.0f, currentBlueTotal = 0.0f, currentAlphaTotal = 0.0f; - // NSUInteger totalNumberOfPixels = round(pixelSizeToUseForTexture.width * pixelSizeToUseForTexture.height); - // - // for (NSUInteger currentPixel = 0; currentPixel < totalNumberOfPixels; currentPixel++) - // { - // currentBlueTotal += (CGFloat)imageData[(currentPixel * 4)] / 255.0f; - // currentGreenTotal += (CGFloat)imageData[(currentPixel * 4) + 1] / 255.0f; - // currentRedTotal += (CGFloat)imageData[(currentPixel * 4 + 2)] / 255.0f; - // currentAlphaTotal += (CGFloat)imageData[(currentPixel * 4) + 3] / 255.0f; - // } - // - // NSLog(@"Debug, average input image red: %f, green: %f, blue: %f, alpha: %f", currentRedTotal / (CGFloat)totalNumberOfPixels, currentGreenTotal / (CGFloat)totalNumberOfPixels, currentBlueTotal / (CGFloat)totalNumberOfPixels, currentAlphaTotal / (CGFloat)totalNumberOfPixels); - - runSynchronouslyOnVideoProcessingQueue(^{ - [GPUImageContext useImageProcessingContext]; - - outputFramebuffer = [[GPUImageContext sharedFramebufferCache] fetchFramebufferForSize:pixelSizeToUseForTexture onlyTexture:YES]; - [outputFramebuffer disableReferenceCounting]; - - glBindTexture(GL_TEXTURE_2D, [outputFramebuffer texture]); - if (self.shouldSmoothlyScaleOutput) - { - glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_LINEAR_MIPMAP_LINEAR); - } - // no need to use self.outputTextureOptions here since pictures need this texture formats and type - glTexImage2D(GL_TEXTURE_2D, 0, GL_RGBA, (int)pixelSizeToUseForTexture.width, (int)pixelSizeToUseForTexture.height, 0, format, GL_UNSIGNED_BYTE, imageData); - - if (self.shouldSmoothlyScaleOutput) - { - glGenerateMipmap(GL_TEXTURE_2D); - } - glBindTexture(GL_TEXTURE_2D, 0); - }); - - if (shouldRedrawUsingCoreGraphics) - { - free(imageData); - } - else - { - CFRelease(dataFromImageDataProvider); - } - - return self; -} - -// ARC forbids explicit message send of 'release' on Mountain Lion, but needs this on Lion and older -#if ( (MAC_OS_X_VERSION_MIN_REQUIRED < __MAC_10_8) || (!defined(__MAC_10_8)) ) -- (void)dealloc; -{ - [outputFramebuffer enableReferenceCounting]; - [outputFramebuffer unlock]; - - if (imageUpdateSemaphore != NULL) - { - dispatch_release(imageUpdateSemaphore); - } -} -#endif - -#pragma mark - -#pragma mark Image rendering - -- (void)removeAllTargets; -{ - [super removeAllTargets]; - hasProcessedImage = NO; -} - -- (void)processImage; -{ - [self processImageWithCompletionHandler:nil]; -} - -- (BOOL)processImageWithCompletionHandler:(void (^)(void))completion; -{ - hasProcessedImage = YES; - - // dispatch_semaphore_wait(imageUpdateSemaphore, DISPATCH_TIME_FOREVER); - - if (dispatch_semaphore_wait(imageUpdateSemaphore, DISPATCH_TIME_NOW) != 0) - { - return NO; - } - - runAsynchronouslyOnVideoProcessingQueue(^{ - for (id currentTarget in targets) - { - NSInteger indexOfObject = [targets indexOfObject:currentTarget]; - NSInteger textureIndexOfTarget = [[targetTextureIndices objectAtIndex:indexOfObject] integerValue]; - - [currentTarget setCurrentlyReceivingMonochromeInput:NO]; - [currentTarget setInputSize:pixelSizeOfImage atIndex:textureIndexOfTarget]; - [currentTarget setInputFramebuffer:outputFramebuffer atIndex:textureIndexOfTarget]; - [currentTarget newFrameReadyAtTime:kCMTimeIndefinite atIndex:textureIndexOfTarget]; - } - - dispatch_semaphore_signal(imageUpdateSemaphore); - - if (completion != nil) { - completion(); - } - }); - - return YES; -} - -- (void)processImageUpToFilter:(GPUImageOutput *)finalFilterInChain withCompletionHandler:(void (^)(NSImage *processedImage))block; -{ - [finalFilterInChain useNextFrameForImageCapture]; - [self processImageWithCompletionHandler:^{ - NSImage *imageFromFilter = [finalFilterInChain imageFromCurrentFramebuffer]; - block(imageFromFilter); - }]; -} - -- (CGSize)outputImageSize; -{ - return pixelSizeOfImage; -} - -- (void)addTarget:(id)newTarget atTextureLocation:(NSInteger)textureLocation; -{ - [super addTarget:newTarget atTextureLocation:textureLocation]; - - if (hasProcessedImage) - { - [newTarget setInputSize:pixelSizeOfImage atIndex:textureLocation]; - [newTarget newFrameReadyAtTime:kCMTimeIndefinite atIndex:textureLocation]; - } -} - -@end \ No newline at end of file diff --git a/ScannerApp/Frameworks/GPUImage/framework/Source/Mac/GPUImageView.h b/ScannerApp/Frameworks/GPUImage/framework/Source/Mac/GPUImageView.h deleted file mode 100755 index 029883a..0000000 --- a/ScannerApp/Frameworks/GPUImage/framework/Source/Mac/GPUImageView.h +++ /dev/null @@ -1,39 +0,0 @@ -#import -#import "GPUImageContext.h" - -typedef enum { - kGPUImageFillModeStretch, // Stretch to fill the full view, which may distort the image outside of its normal aspect ratio - kGPUImageFillModePreserveAspectRatio, // Maintains the aspect ratio of the source image, adding bars of the specified background color - kGPUImageFillModePreserveAspectRatioAndFill // Maintains the aspect ratio of the source image, zooming in on its center to fill the view -} GPUImageFillModeType; - -/** - UIView subclass to use as an endpoint for displaying GPUImage outputs - */ -@interface GPUImageView : NSOpenGLView -{ - GPUImageRotationMode inputRotation; -} - -/** The fill mode dictates how images are fit in the view, with the default being kGPUImageFillModePreserveAspectRatio - */ -@property(readwrite, nonatomic) GPUImageFillModeType fillMode; - -/** This calculates the current display size, in pixels, taking into account Retina scaling factors - */ -@property(readonly, nonatomic) CGSize sizeInPixels; - -@property(nonatomic) BOOL enabled; - -/** Handling fill mode - - @param redComponent Red component for background color - @param greenComponent Green component for background color - @param blueComponent Blue component for background color - @param alphaComponent Alpha component for background color - */ -- (void)setBackgroundColorRed:(GLfloat)redComponent green:(GLfloat)greenComponent blue:(GLfloat)blueComponent alpha:(GLfloat)alphaComponent; - -- (void)setCurrentlyReceivingMonochromeInput:(BOOL)newValue; - -@end diff --git a/ScannerApp/Frameworks/GPUImage/framework/Source/Mac/GPUImageView.m b/ScannerApp/Frameworks/GPUImage/framework/Source/Mac/GPUImageView.m deleted file mode 100755 index 9e277ac..0000000 --- a/ScannerApp/Frameworks/GPUImage/framework/Source/Mac/GPUImageView.m +++ /dev/null @@ -1,458 +0,0 @@ -#import "GPUImageView.h" -#import -#import "GPUImageContext.h" -#import "GPUImageFilter.h" -#import - -#pragma mark - -#pragma mark Private methods and instance variables - -@interface GPUImageView () -{ - GPUImageFramebuffer *inputFramebufferForDisplay; - - GLProgram *displayProgram; - GLint displayPositionAttribute, displayTextureCoordinateAttribute; - GLint displayInputTextureUniform; - - CGSize inputImageSize; - GLfloat imageVertices[8]; - GLfloat backgroundColorRed, backgroundColorGreen, backgroundColorBlue, backgroundColorAlpha; -} - -// Initialization and teardown -- (void)commonInit; - -// Managing the display FBOs -- (void)createDisplayFramebuffer; -- (void)destroyDisplayFramebuffer; - -// Handling fill mode -- (void)recalculateViewGeometry; - -@end - -@implementation GPUImageView - -@synthesize sizeInPixels = _sizeInPixels; -@synthesize fillMode = _fillMode; -@synthesize enabled; - -#pragma mark - -#pragma mark Initialization and teardown - -- (id)initWithFrame:(CGRect)frame -{ - if (!(self = [super initWithFrame:frame])) - { - return nil; - } - - [self commonInit]; - - return self; -} - --(id)initWithCoder:(NSCoder *)coder -{ - if (!(self = [super initWithCoder:coder])) - { - return nil; - } - - [self commonInit]; - - return self; -} - -- (void)commonInit; -{ - [self setOpenGLContext:[[GPUImageContext sharedImageProcessingContext] context]]; - - if ([self respondsToSelector:@selector(setWantsBestResolutionOpenGLSurface:)]) - { - [self setWantsBestResolutionOpenGLSurface:YES]; - } - - inputRotation = kGPUImageNoRotation; - self.hidden = NO; - - self.enabled = YES; - - runSynchronouslyOnVideoProcessingQueue(^{ - [GPUImageContext useImageProcessingContext]; - displayProgram = [[GPUImageContext sharedImageProcessingContext] programForVertexShaderString:kGPUImageVertexShaderString fragmentShaderString:kGPUImagePassthroughFragmentShaderString]; - - if (!displayProgram.initialized) - { - [displayProgram addAttribute:@"position"]; - [displayProgram addAttribute:@"inputTextureCoordinate"]; - - if (![displayProgram link]) - { - NSString *progLog = [displayProgram programLog]; - NSLog(@"Program link log: %@", progLog); - NSString *fragLog = [displayProgram fragmentShaderLog]; - NSLog(@"Fragment shader compile log: %@", fragLog); - NSString *vertLog = [displayProgram vertexShaderLog]; - NSLog(@"Vertex shader compile log: %@", vertLog); - displayProgram = nil; - NSAssert(NO, @"Filter shader link failed"); - } - } - - displayPositionAttribute = [displayProgram attributeIndex:@"position"]; - displayTextureCoordinateAttribute = [displayProgram attributeIndex:@"inputTextureCoordinate"]; - displayInputTextureUniform = [displayProgram uniformIndex:@"inputImageTexture"]; - - [GPUImageContext setActiveShaderProgram:displayProgram]; - - glEnableVertexAttribArray(displayPositionAttribute); - glEnableVertexAttribArray(displayTextureCoordinateAttribute); - - [self setBackgroundColorRed:0.0 green:0.0 blue:0.0 alpha:1.0]; - _fillMode = kGPUImageFillModePreserveAspectRatio; - [self createDisplayFramebuffer]; - }); - -} - -- (void)dealloc -{ -} - -#pragma mark - -#pragma mark Managing the display FBOs - -- (void)createDisplayFramebuffer; -{ - // Perhaps I'll use an FBO at some time later, but for now will render directly to the screen - if ([self respondsToSelector:@selector(convertSizeToBacking:)]) - { - _sizeInPixels = [self convertSizeToBacking:self.bounds.size]; - } - else - { - _sizeInPixels = self.bounds.size; - } -} - -- (void)destroyDisplayFramebuffer; -{ - [self.openGLContext makeCurrentContext]; -} - -- (void)setDisplayFramebuffer; -{ - glBindFramebuffer(GL_FRAMEBUFFER, 0); - glBindRenderbuffer(GL_RENDERBUFFER, 0); - - glViewport(0, 0, (GLint)_sizeInPixels.width, (GLint)_sizeInPixels.height); -} - -- (void)presentFramebuffer; -{ - [self.openGLContext flushBuffer]; -} - -- (void)reshape; -{ - CGSize viewSize = self.bounds.size; - if ([self respondsToSelector:@selector(convertSizeToBacking:)]) - { - viewSize = [self convertSizeToBacking:self.bounds.size]; - } - - if ( (_sizeInPixels.width == viewSize.width) && (_sizeInPixels.height == viewSize.height) ) - { - return; - } - - _sizeInPixels = viewSize; - - [self recalculateViewGeometry]; - dispatch_async(dispatch_get_global_queue(DISPATCH_QUEUE_PRIORITY_DEFAULT, 0), ^{ - [self newFrameReadyAtTime:kCMTimeInvalid atIndex:0]; - }); -} - -#pragma mark - -#pragma mark Handling fill mode - -- (void)recalculateViewGeometry; -{ - CGFloat heightScaling, widthScaling; - - CGSize currentViewSize = self.sizeInPixels; - - if ((inputImageSize.width < 1.0) || (inputImageSize.height < 1.0)) - { - return; - } - - CGRect insetRect = AVMakeRectWithAspectRatioInsideRect(inputImageSize, CGRectMake(0.0,0.0,currentViewSize.width,currentViewSize.height)); - if ((insetRect.size.width < 1.0) || (insetRect.size.width < 1.0)) - { - insetRect = CGRectMake(0.0,0.0,currentViewSize.width,currentViewSize.height); - } - - switch(_fillMode) - { - case kGPUImageFillModeStretch: - { - widthScaling = 1.0; - heightScaling = 1.0; - }; break; - case kGPUImageFillModePreserveAspectRatio: - { - widthScaling = insetRect.size.width / currentViewSize.width; - heightScaling = insetRect.size.height / currentViewSize.height; - }; break; - case kGPUImageFillModePreserveAspectRatioAndFill: - { - widthScaling = currentViewSize.height / insetRect.size.height; - heightScaling = currentViewSize.width / insetRect.size.width; - }; break; - } - - imageVertices[0] = -widthScaling; - imageVertices[1] = -heightScaling; - imageVertices[2] = widthScaling; - imageVertices[3] = -heightScaling; - imageVertices[4] = -widthScaling; - imageVertices[5] = heightScaling; - imageVertices[6] = widthScaling; - imageVertices[7] = heightScaling; -} - -- (void)setBackgroundColorRed:(GLfloat)redComponent green:(GLfloat)greenComponent blue:(GLfloat)blueComponent alpha:(GLfloat)alphaComponent; -{ - backgroundColorRed = redComponent; - backgroundColorGreen = greenComponent; - backgroundColorBlue = blueComponent; - backgroundColorAlpha = alphaComponent; -} - -+ (const GLfloat *)textureCoordinatesForRotation:(GPUImageRotationMode)rotationMode; -{ -// static const GLfloat noRotationTextureCoordinates[] = { -// 0.0f, 0.0f, -// 1.0f, 0.0f, -// 0.0f, 1.0f, -// 1.0f, 1.0f, -// }; - - static const GLfloat noRotationTextureCoordinates[] = { - 0.0f, 1.0f, - 1.0f, 1.0f, - 0.0f, 0.0f, - 1.0f, 0.0f, - }; - - static const GLfloat rotateRightTextureCoordinates[] = { - 1.0f, 1.0f, - 1.0f, 0.0f, - 0.0f, 1.0f, - 0.0f, 0.0f, - }; - - static const GLfloat rotateLeftTextureCoordinates[] = { - 0.0f, 0.0f, - 0.0f, 1.0f, - 1.0f, 0.0f, - 1.0f, 1.0f, - }; - - static const GLfloat verticalFlipTextureCoordinates[] = { - 0.0f, 0.0f, - 1.0f, 0.0f, - 0.0f, 1.0f, - 1.0f, 1.0f, - }; - - static const GLfloat horizontalFlipTextureCoordinates[] = { - 1.0f, 1.0f, - 0.0f, 1.0f, - 1.0f, 0.0f, - 0.0f, 0.0f, - }; - - static const GLfloat rotateRightVerticalFlipTextureCoordinates[] = { - 1.0f, 0.0f, - 1.0f, 1.0f, - 0.0f, 0.0f, - 0.0f, 1.0f, - }; - - static const GLfloat rotateRightHorizontalFlipTextureCoordinates[] = { - 1.0f, 1.0f, - 1.0f, 0.0f, - 0.0f, 1.0f, - 0.0f, 0.0f, - }; - - static const GLfloat rotate180TextureCoordinates[] = { - 1.0f, 0.0f, - 0.0f, 0.0f, - 1.0f, 1.0f, - 0.0f, 1.0f, - }; - - switch(rotationMode) - { - case kGPUImageNoRotation: return noRotationTextureCoordinates; - case kGPUImageRotateLeft: return rotateLeftTextureCoordinates; - case kGPUImageRotateRight: return rotateRightTextureCoordinates; - case kGPUImageFlipVertical: return verticalFlipTextureCoordinates; - case kGPUImageFlipHorizonal: return horizontalFlipTextureCoordinates; - case kGPUImageRotateRightFlipVertical: return rotateRightVerticalFlipTextureCoordinates; - case kGPUImageRotateRightFlipHorizontal: return rotateRightHorizontalFlipTextureCoordinates; - case kGPUImageRotate180: return rotate180TextureCoordinates; - } -} - -#pragma mark - -#pragma mark GPUInput protocol - -- (void)newFrameReadyAtTime:(CMTime)frameTime atIndex:(NSInteger)textureIndex; -{ - runSynchronouslyOnVideoProcessingQueue(^{ - [GPUImageContext setActiveShaderProgram:displayProgram]; - [self setDisplayFramebuffer]; - - glClearColor(backgroundColorRed, backgroundColorGreen, backgroundColorBlue, backgroundColorAlpha); - glClear(GL_COLOR_BUFFER_BIT); - - // Re-render onscreen, flipped to a normal orientation - glBindFramebuffer(GL_FRAMEBUFFER, 0); - glBindRenderbuffer(GL_RENDERBUFFER, 0); - - glActiveTexture(GL_TEXTURE4); - glBindTexture(GL_TEXTURE_2D, [inputFramebufferForDisplay texture]); - glUniform1i(displayInputTextureUniform, 4); - - glVertexAttribPointer(displayPositionAttribute, 2, GL_FLOAT, 0, 0, imageVertices); - glVertexAttribPointer(displayTextureCoordinateAttribute, 2, GL_FLOAT, 0, 0, [GPUImageView textureCoordinatesForRotation:inputRotation]); - - BOOL canLockFocus = YES; - if ([self respondsToSelector:@selector(lockFocusIfCanDraw)]) - { - canLockFocus = [self lockFocusIfCanDraw]; - } - else - { - [self lockFocus]; - } - - if (canLockFocus) - { - glDrawArrays(GL_TRIANGLE_STRIP, 0, 4); - - [self presentFramebuffer]; - glBindTexture(GL_TEXTURE_2D, 0); - [self unlockFocus]; - } - - [inputFramebufferForDisplay unlock]; - inputFramebufferForDisplay = nil; - }); -} - -- (NSInteger)nextAvailableTextureIndex; -{ - return 0; -} - -- (void)setInputFramebuffer:(GPUImageFramebuffer *)newInputFramebuffer atIndex:(NSInteger)textureIndex; -{ - inputFramebufferForDisplay = newInputFramebuffer; - [inputFramebufferForDisplay lock]; -} - -- (void)setInputRotation:(GPUImageRotationMode)newInputRotation atIndex:(NSInteger)textureIndex; -{ - inputRotation = newInputRotation; -} - -- (void)setInputSize:(CGSize)newSize atIndex:(NSInteger)textureIndex; -{ - if ((newSize.width < 1.0) || (newSize.height < 1.0)) - { - return; - } - - runSynchronouslyOnVideoProcessingQueue(^{ - CGSize rotatedSize = newSize; - - if (GPUImageRotationSwapsWidthAndHeight(inputRotation)) - { - rotatedSize.width = newSize.height; - rotatedSize.height = newSize.width; - } - - if (!CGSizeEqualToSize(inputImageSize, rotatedSize)) - { - inputImageSize = rotatedSize; - [self recalculateViewGeometry]; - } - }); -} - -- (CGSize)maximumOutputSize; -{ - if ([self respondsToSelector:@selector(convertSizeToBacking:)]) - { - return [self convertSizeToBacking:self.bounds.size]; - } - else - { - return self.bounds.size; - } -} - -- (void)endProcessing -{ -} - -- (BOOL)shouldIgnoreUpdatesToThisTarget; -{ - return NO; -} - -- (void)conserveMemoryForNextFrame; -{ - -} - -- (BOOL)wantsMonochromeInput; -{ - return NO; -} - -- (void)setCurrentlyReceivingMonochromeInput:(BOOL)newValue; -{ - -} - -#pragma mark - -#pragma mark Accessors - -- (CGSize)sizeInPixels; -{ - if (CGSizeEqualToSize(_sizeInPixels, CGSizeZero)) - { - return [self maximumOutputSize]; - } - else - { - return _sizeInPixels; - } -} - -- (void)setFillMode:(GPUImageFillModeType)newValue; -{ - _fillMode = newValue; - [self recalculateViewGeometry]; -} - -@end diff --git a/ScannerApp/Frameworks/GPUImage/framework/Source/Mac/en.lproj/InfoPlist.strings b/ScannerApp/Frameworks/GPUImage/framework/Source/Mac/en.lproj/InfoPlist.strings deleted file mode 100644 index 477b28f..0000000 --- a/ScannerApp/Frameworks/GPUImage/framework/Source/Mac/en.lproj/InfoPlist.strings +++ /dev/null @@ -1,2 +0,0 @@ -/* Localized versions of Info.plist keys */ - diff --git a/ScannerApp/Frameworks/GPUImage/framework/Source/iOS/GPUImageMovieWriter.m b/ScannerApp/Frameworks/GPUImage/framework/Source/iOS/GPUImageMovieWriter.m index addc0a7..1974d61 100755 --- a/ScannerApp/Frameworks/GPUImage/framework/Source/iOS/GPUImageMovieWriter.m +++ b/ScannerApp/Frameworks/GPUImage/framework/Source/iOS/GPUImageMovieWriter.m @@ -101,43 +101,43 @@ - (id)initWithMovieURL:(NSURL *)newMovieURL size:(CGSize)newSize fileType:(NSStr [_movieWriterContext useSharegroup:[[[GPUImageContext sharedImageProcessingContext] context] sharegroup]]; runSynchronouslyOnContextQueue(_movieWriterContext, ^{ - [_movieWriterContext useAsCurrentContext]; + [self->_movieWriterContext useAsCurrentContext]; if ([GPUImageContext supportsFastTextureUpload]) { - colorSwizzlingProgram = [_movieWriterContext programForVertexShaderString:kGPUImageVertexShaderString fragmentShaderString:kGPUImagePassthroughFragmentShaderString]; + self->colorSwizzlingProgram = [self->_movieWriterContext programForVertexShaderString:kGPUImageVertexShaderString fragmentShaderString:kGPUImagePassthroughFragmentShaderString]; } else { - colorSwizzlingProgram = [_movieWriterContext programForVertexShaderString:kGPUImageVertexShaderString fragmentShaderString:kGPUImageColorSwizzlingFragmentShaderString]; + self->colorSwizzlingProgram = [self->_movieWriterContext programForVertexShaderString:kGPUImageVertexShaderString fragmentShaderString:kGPUImageColorSwizzlingFragmentShaderString]; } - if (!colorSwizzlingProgram.initialized) + if (!self->colorSwizzlingProgram.initialized) { - [colorSwizzlingProgram addAttribute:@"position"]; - [colorSwizzlingProgram addAttribute:@"inputTextureCoordinate"]; + [self->colorSwizzlingProgram addAttribute:@"position"]; + [self->colorSwizzlingProgram addAttribute:@"inputTextureCoordinate"]; - if (![colorSwizzlingProgram link]) + if (![self->colorSwizzlingProgram link]) { - NSString *progLog = [colorSwizzlingProgram programLog]; + NSString *progLog = [self->colorSwizzlingProgram programLog]; NSLog(@"Program link log: %@", progLog); - NSString *fragLog = [colorSwizzlingProgram fragmentShaderLog]; + NSString *fragLog = [self->colorSwizzlingProgram fragmentShaderLog]; NSLog(@"Fragment shader compile log: %@", fragLog); - NSString *vertLog = [colorSwizzlingProgram vertexShaderLog]; + NSString *vertLog = [self->colorSwizzlingProgram vertexShaderLog]; NSLog(@"Vertex shader compile log: %@", vertLog); - colorSwizzlingProgram = nil; + self->colorSwizzlingProgram = nil; NSAssert(NO, @"Filter shader link failed"); } } - colorSwizzlingPositionAttribute = [colorSwizzlingProgram attributeIndex:@"position"]; - colorSwizzlingTextureCoordinateAttribute = [colorSwizzlingProgram attributeIndex:@"inputTextureCoordinate"]; - colorSwizzlingInputTextureUniform = [colorSwizzlingProgram uniformIndex:@"inputImageTexture"]; + self->colorSwizzlingPositionAttribute = [self->colorSwizzlingProgram attributeIndex:@"position"]; + self->colorSwizzlingTextureCoordinateAttribute = [self->colorSwizzlingProgram attributeIndex:@"inputTextureCoordinate"]; + self->colorSwizzlingInputTextureUniform = [self->colorSwizzlingProgram uniformIndex:@"inputImageTexture"]; - [_movieWriterContext setContextShaderProgram:colorSwizzlingProgram]; + [self->_movieWriterContext setContextShaderProgram:self->colorSwizzlingProgram]; - glEnableVertexAttribArray(colorSwizzlingPositionAttribute); - glEnableVertexAttribArray(colorSwizzlingTextureCoordinateAttribute); + glEnableVertexAttribArray(self->colorSwizzlingPositionAttribute); + glEnableVertexAttribArray(self->colorSwizzlingTextureCoordinateAttribute); }); [self initializeMovieWithOutputSettings:outputSettings]; @@ -272,9 +272,9 @@ - (void)startRecording; alreadyFinishedRecording = NO; startTime = kCMTimeInvalid; runSynchronouslyOnContextQueue(_movieWriterContext, ^{ - if (audioInputReadyCallback == NULL) + if (self->audioInputReadyCallback == NULL) { - [assetWriter startWriting]; + [self->assetWriter startWriting]; } }); isRecording = YES; @@ -297,19 +297,19 @@ - (void)cancelRecording; isRecording = NO; runSynchronouslyOnContextQueue(_movieWriterContext, ^{ - alreadyFinishedRecording = YES; + self->alreadyFinishedRecording = YES; - if( assetWriter.status == AVAssetWriterStatusWriting && ! videoEncodingIsFinished ) + if( self->assetWriter.status == AVAssetWriterStatusWriting && ! self->videoEncodingIsFinished ) { - videoEncodingIsFinished = YES; - [assetWriterVideoInput markAsFinished]; + self->videoEncodingIsFinished = YES; + [self->assetWriterVideoInput markAsFinished]; } - if( assetWriter.status == AVAssetWriterStatusWriting && ! audioEncodingIsFinished ) + if( self->assetWriter.status == AVAssetWriterStatusWriting && ! self->audioEncodingIsFinished ) { - audioEncodingIsFinished = YES; - [assetWriterAudioInput markAsFinished]; + self->audioEncodingIsFinished = YES; + [self->assetWriterAudioInput markAsFinished]; } - [assetWriter cancelWriting]; + [self->assetWriter cancelWriting]; }); } @@ -321,23 +321,23 @@ - (void)finishRecording; - (void)finishRecordingWithCompletionHandler:(void (^)(void))handler; { runSynchronouslyOnContextQueue(_movieWriterContext, ^{ - isRecording = NO; + self->isRecording = NO; - if (assetWriter.status == AVAssetWriterStatusCompleted || assetWriter.status == AVAssetWriterStatusCancelled || assetWriter.status == AVAssetWriterStatusUnknown) + if (self->assetWriter.status == AVAssetWriterStatusCompleted || self->assetWriter.status == AVAssetWriterStatusCancelled || self->assetWriter.status == AVAssetWriterStatusUnknown) { if (handler) - runAsynchronouslyOnContextQueue(_movieWriterContext, handler); + runAsynchronouslyOnContextQueue(self->_movieWriterContext, handler); return; } - if( assetWriter.status == AVAssetWriterStatusWriting && ! videoEncodingIsFinished ) + if( self->assetWriter.status == AVAssetWriterStatusWriting && ! self->videoEncodingIsFinished ) { - videoEncodingIsFinished = YES; - [assetWriterVideoInput markAsFinished]; + self->videoEncodingIsFinished = YES; + [self->assetWriterVideoInput markAsFinished]; } - if( assetWriter.status == AVAssetWriterStatusWriting && ! audioEncodingIsFinished ) + if( self->assetWriter.status == AVAssetWriterStatusWriting && ! self->audioEncodingIsFinished ) { - audioEncodingIsFinished = YES; - [assetWriterAudioInput markAsFinished]; + self->audioEncodingIsFinished = YES; + [self->assetWriterAudioInput markAsFinished]; } #if (!defined(__IPHONE_6_0) || (__IPHONE_OS_VERSION_MAX_ALLOWED < __IPHONE_6_0)) // Not iOS 6 SDK @@ -346,18 +346,18 @@ - (void)finishRecordingWithCompletionHandler:(void (^)(void))handler; runAsynchronouslyOnContextQueue(_movieWriterContext,handler); #else // iOS 6 SDK - if ([assetWriter respondsToSelector:@selector(finishWritingWithCompletionHandler:)]) { + if ([self->assetWriter respondsToSelector:@selector(finishWritingWithCompletionHandler:)]) { // Running iOS 6 - [assetWriter finishWritingWithCompletionHandler:(handler ?: ^{ })]; + [self->assetWriter finishWritingWithCompletionHandler:(handler ?: ^{ })]; } else { // Not running iOS 6 #pragma clang diagnostic push #pragma clang diagnostic ignored "-Wdeprecated-declarations" - [assetWriter finishWriting]; + [self->assetWriter finishWriting]; #pragma clang diagnostic pop if (handler) - runAsynchronouslyOnContextQueue(_movieWriterContext, handler); + runAsynchronouslyOnContextQueue(self->_movieWriterContext, handler); } #endif }); @@ -380,12 +380,12 @@ - (void)processAudioBuffer:(CMSampleBufferRef)audioBuffer; if (CMTIME_IS_INVALID(startTime)) { runSynchronouslyOnContextQueue(_movieWriterContext, ^{ - if ((audioInputReadyCallback == NULL) && (assetWriter.status != AVAssetWriterStatusWriting)) + if ((self->audioInputReadyCallback == NULL) && (self->assetWriter.status != AVAssetWriterStatusWriting)) { - [assetWriter startWriting]; + [self->assetWriter startWriting]; } - [assetWriter startSessionAtSourceTime:currentSampleTime]; - startTime = currentSampleTime; + [self->assetWriter startSessionAtSourceTime:currentSampleTime]; + self->startTime = currentSampleTime; }); } @@ -454,19 +454,19 @@ - (void)processAudioBuffer:(CMSampleBufferRef)audioBuffer; } // NSLog(@"Recorded audio sample time: %lld, %d, %lld", currentSampleTime.value, currentSampleTime.timescale, currentSampleTime.epoch); - void(^write)() = ^() { - while( ! assetWriterAudioInput.readyForMoreMediaData && ! _encodingLiveVideo && ! audioEncodingIsFinished ) { + void(^write)(void) = ^() { + while( ! self->assetWriterAudioInput.readyForMoreMediaData && ! self->_encodingLiveVideo && ! self->audioEncodingIsFinished ) { NSDate *maxDate = [NSDate dateWithTimeIntervalSinceNow:0.5]; //NSLog(@"audio waiting..."); [[NSRunLoop currentRunLoop] runUntilDate:maxDate]; } - if (!assetWriterAudioInput.readyForMoreMediaData) + if (!self->assetWriterAudioInput.readyForMoreMediaData) { NSLog(@"2: Had to drop an audio frame %@", CFBridgingRelease(CMTimeCopyDescription(kCFAllocatorDefault, currentSampleTime))); } - else if(assetWriter.status == AVAssetWriterStatusWriting) + else if(self->assetWriter.status == AVAssetWriterStatusWriting) { - if (![assetWriterAudioInput appendSampleBuffer:audioBuffer]) + if (![self->assetWriterAudioInput appendSampleBuffer:audioBuffer]) NSLog(@"Problem appending audio buffer at time: %@", CFBridgingRelease(CMTimeCopyDescription(kCFAllocatorDefault, currentSampleTime))); } else @@ -474,7 +474,7 @@ - (void)processAudioBuffer:(CMSampleBufferRef)audioBuffer; //NSLog(@"Wrote an audio frame %@", CFBridgingRelease(CMTimeCopyDescription(kCFAllocatorDefault, currentSampleTime))); } - if (_shouldInvalidateAudioSampleWhenDone) + if (self->_shouldInvalidateAudioSampleWhenDone) { CMSampleBufferInvalidate(audioBuffer); } @@ -503,7 +503,7 @@ - (void)enableSynchronizationCallbacks; } videoQueue = dispatch_queue_create("com.sunsetlakesoftware.GPUImage.videoReadingQueue", GPUImageDefaultQueueAttribute()); [assetWriterVideoInput requestMediaDataWhenReadyOnQueue:videoQueue usingBlock:^{ - if( _paused ) + if( self->_paused ) { //NSLog(@"video requestMediaDataWhenReadyOnQueue paused"); // if we don't sleep, we'll get called back almost immediately, chewing up CPU @@ -511,15 +511,15 @@ - (void)enableSynchronizationCallbacks; return; } //NSLog(@"video requestMediaDataWhenReadyOnQueue begin"); - while( assetWriterVideoInput.readyForMoreMediaData && ! _paused ) + while( self->assetWriterVideoInput.readyForMoreMediaData && ! self->_paused ) { - if( videoInputReadyCallback && ! videoInputReadyCallback() && ! videoEncodingIsFinished ) + if( self->videoInputReadyCallback && ! self->videoInputReadyCallback() && ! self->videoEncodingIsFinished ) { - runAsynchronouslyOnContextQueue(_movieWriterContext, ^{ - if( assetWriter.status == AVAssetWriterStatusWriting && ! videoEncodingIsFinished ) + runAsynchronouslyOnContextQueue(self->_movieWriterContext, ^{ + if( self->assetWriter.status == AVAssetWriterStatusWriting && ! self->videoEncodingIsFinished ) { - videoEncodingIsFinished = YES; - [assetWriterVideoInput markAsFinished]; + self->videoEncodingIsFinished = YES; + [self->assetWriterVideoInput markAsFinished]; } }); } @@ -532,7 +532,7 @@ - (void)enableSynchronizationCallbacks; { audioQueue = dispatch_queue_create("com.sunsetlakesoftware.GPUImage.audioReadingQueue", GPUImageDefaultQueueAttribute()); [assetWriterAudioInput requestMediaDataWhenReadyOnQueue:audioQueue usingBlock:^{ - if( _paused ) + if( self->_paused ) { //NSLog(@"audio requestMediaDataWhenReadyOnQueue paused"); // if we don't sleep, we'll get called back almost immediately, chewing up CPU @@ -540,15 +540,15 @@ - (void)enableSynchronizationCallbacks; return; } //NSLog(@"audio requestMediaDataWhenReadyOnQueue begin"); - while( assetWriterAudioInput.readyForMoreMediaData && ! _paused ) + while( self->assetWriterAudioInput.readyForMoreMediaData && ! self->_paused ) { - if( audioInputReadyCallback && ! audioInputReadyCallback() && ! audioEncodingIsFinished ) + if( self->audioInputReadyCallback && ! self->audioInputReadyCallback() && ! self->audioEncodingIsFinished ) { - runAsynchronouslyOnContextQueue(_movieWriterContext, ^{ - if( assetWriter.status == AVAssetWriterStatusWriting && ! audioEncodingIsFinished ) + runAsynchronouslyOnContextQueue(self->_movieWriterContext, ^{ + if( self->assetWriter.status == AVAssetWriterStatusWriting && ! self->audioEncodingIsFinished ) { - audioEncodingIsFinished = YES; - [assetWriterAudioInput markAsFinished]; + self->audioEncodingIsFinished = YES; + [self->assetWriterAudioInput markAsFinished]; } }); } @@ -618,29 +618,29 @@ - (void)createDataFBO; - (void)destroyDataFBO; { runSynchronouslyOnContextQueue(_movieWriterContext, ^{ - [_movieWriterContext useAsCurrentContext]; + [self->_movieWriterContext useAsCurrentContext]; - if (movieFramebuffer) + if (self->movieFramebuffer) { - glDeleteFramebuffers(1, &movieFramebuffer); - movieFramebuffer = 0; + glDeleteFramebuffers(1, &self->movieFramebuffer); + self->movieFramebuffer = 0; } - if (movieRenderbuffer) + if (self->movieRenderbuffer) { - glDeleteRenderbuffers(1, &movieRenderbuffer); - movieRenderbuffer = 0; + glDeleteRenderbuffers(1, &self->movieRenderbuffer); + self->movieRenderbuffer = 0; } if ([GPUImageContext supportsFastTextureUpload]) { - if (renderTexture) + if (self->renderTexture) { - CFRelease(renderTexture); + CFRelease(self->renderTexture); } - if (renderTarget) + if (self->renderTarget) { - CVPixelBufferRelease(renderTarget); + CVPixelBufferRelease(self->renderTarget); } } @@ -737,13 +737,13 @@ - (void)newFrameReadyAtTime:(CMTime)frameTime atIndex:(NSInteger)textureIndex; if (CMTIME_IS_INVALID(startTime)) { runSynchronouslyOnContextQueue(_movieWriterContext, ^{ - if ((videoInputReadyCallback == NULL) && (assetWriter.status != AVAssetWriterStatusWriting)) + if ((self->videoInputReadyCallback == NULL) && (self->assetWriter.status != AVAssetWriterStatusWriting)) { - [assetWriter startWriting]; + [self->assetWriter startWriting]; } - [assetWriter startSessionAtSourceTime:frameTime]; - startTime = frameTime; + [self->assetWriter startSessionAtSourceTime:frameTime]; + self->startTime = frameTime; }); } @@ -751,7 +751,7 @@ - (void)newFrameReadyAtTime:(CMTime)frameTime atIndex:(NSInteger)textureIndex; glFinish(); runAsynchronouslyOnContextQueue(_movieWriterContext, ^{ - if (!assetWriterVideoInput.readyForMoreMediaData && _encodingLiveVideo) + if (!self->assetWriterVideoInput.readyForMoreMediaData && self->_encodingLiveVideo) { [inputFramebufferForBlock unlock]; NSLog(@"1: Had to drop a video frame: %@", CFBridgingRelease(CMTimeCopyDescription(kCFAllocatorDefault, frameTime))); @@ -759,19 +759,19 @@ - (void)newFrameReadyAtTime:(CMTime)frameTime atIndex:(NSInteger)textureIndex; } // Render the frame with swizzled colors, so that they can be uploaded quickly as BGRA frames - [_movieWriterContext useAsCurrentContext]; + [self->_movieWriterContext useAsCurrentContext]; [self renderAtInternalSizeUsingFramebuffer:inputFramebufferForBlock]; CVPixelBufferRef pixel_buffer = NULL; if ([GPUImageContext supportsFastTextureUpload]) { - pixel_buffer = renderTarget; + pixel_buffer = self->renderTarget; CVPixelBufferLockBaseAddress(pixel_buffer, 0); } else { - CVReturn status = CVPixelBufferPoolCreatePixelBuffer (NULL, [assetWriterPixelBufferInput pixelBufferPool], &pixel_buffer); + CVReturn status = CVPixelBufferPoolCreatePixelBuffer (NULL, [self->assetWriterPixelBufferInput pixelBufferPool], &pixel_buffer); if ((pixel_buffer == NULL) || (status != kCVReturnSuccess)) { CVPixelBufferRelease(pixel_buffer); @@ -782,23 +782,23 @@ - (void)newFrameReadyAtTime:(CMTime)frameTime atIndex:(NSInteger)textureIndex; CVPixelBufferLockBaseAddress(pixel_buffer, 0); GLubyte *pixelBufferData = (GLubyte *)CVPixelBufferGetBaseAddress(pixel_buffer); - glReadPixels(0, 0, videoSize.width, videoSize.height, GL_RGBA, GL_UNSIGNED_BYTE, pixelBufferData); + glReadPixels(0, 0, self->videoSize.width, self->videoSize.height, GL_RGBA, GL_UNSIGNED_BYTE, pixelBufferData); } } - void(^write)() = ^() { - while( ! assetWriterVideoInput.readyForMoreMediaData && ! _encodingLiveVideo && ! videoEncodingIsFinished ) { + void(^write)(void) = ^() { + while( ! self->assetWriterVideoInput.readyForMoreMediaData && ! self->_encodingLiveVideo && ! self->videoEncodingIsFinished ) { NSDate *maxDate = [NSDate dateWithTimeIntervalSinceNow:0.1]; // NSLog(@"video waiting..."); [[NSRunLoop currentRunLoop] runUntilDate:maxDate]; } - if (!assetWriterVideoInput.readyForMoreMediaData) + if (!self->assetWriterVideoInput.readyForMoreMediaData) { NSLog(@"2: Had to drop a video frame: %@", CFBridgingRelease(CMTimeCopyDescription(kCFAllocatorDefault, frameTime))); } else if(self.assetWriter.status == AVAssetWriterStatusWriting) { - if (![assetWriterPixelBufferInput appendPixelBuffer:pixel_buffer withPresentationTime:frameTime]) + if (![self->assetWriterPixelBufferInput appendPixelBuffer:pixel_buffer withPresentationTime:frameTime]) NSLog(@"Problem appending pixel buffer at time: %@", CFBridgingRelease(CMTimeCopyDescription(kCFAllocatorDefault, frameTime))); } else @@ -808,7 +808,7 @@ - (void)newFrameReadyAtTime:(CMTime)frameTime atIndex:(NSInteger)textureIndex; } CVPixelBufferUnlockBaseAddress(pixel_buffer, 0); - previousFrameTime = frameTime; + self->previousFrameTime = frameTime; if (![GPUImageContext supportsFastTextureUpload]) { diff --git a/ScannerApp/Frameworks/GPUImage/framework/Source/iOS/GPUImagePicture+TextureSubimage.m b/ScannerApp/Frameworks/GPUImage/framework/Source/iOS/GPUImagePicture+TextureSubimage.m index 71ef8f9..aa45b66 100644 --- a/ScannerApp/Frameworks/GPUImage/framework/Source/iOS/GPUImagePicture+TextureSubimage.m +++ b/ScannerApp/Frameworks/GPUImage/framework/Source/iOS/GPUImagePicture+TextureSubimage.m @@ -77,9 +77,9 @@ - (void)replaceTextureWithSubCGImage:(CGImageRef)subimageSource inRect:(CGRect)s runSynchronouslyOnVideoProcessingQueue(^{ [GPUImageContext useImageProcessingContext]; - [outputFramebuffer disableReferenceCounting]; + [self->outputFramebuffer disableReferenceCounting]; - glBindTexture(GL_TEXTURE_2D, [outputFramebuffer texture]); + glBindTexture(GL_TEXTURE_2D, [self->outputFramebuffer texture]); // no need to use self.outputTextureOptions here since pictures need this texture formats and type glTexSubImage2D(GL_TEXTURE_2D, 0, subRect.origin.x, subRect.origin.y, (GLint)subRect.size.width, subRect.size.height, GL_RGBA, GL_UNSIGNED_BYTE, imageData); diff --git a/ScannerApp/Frameworks/GPUImage/framework/Source/iOS/GPUImagePicture.m b/ScannerApp/Frameworks/GPUImage/framework/Source/iOS/GPUImagePicture.m index c525f4d..184e851 100755 --- a/ScannerApp/Frameworks/GPUImage/framework/Source/iOS/GPUImagePicture.m +++ b/ScannerApp/Frameworks/GPUImage/framework/Source/iOS/GPUImagePicture.m @@ -249,10 +249,10 @@ - (id)initWithCGImage:(CGImageRef)newImageSource smoothlyScaleOutput:(BOOL)smoot runSynchronouslyOnVideoProcessingQueue(^{ [GPUImageContext useImageProcessingContext]; - outputFramebuffer = [[GPUImageContext sharedFramebufferCache] fetchFramebufferForSize:pixelSizeToUseForTexture onlyTexture:YES]; - [outputFramebuffer disableReferenceCounting]; + self->outputFramebuffer = [[GPUImageContext sharedFramebufferCache] fetchFramebufferForSize:pixelSizeToUseForTexture onlyTexture:YES]; + [self->outputFramebuffer disableReferenceCounting]; - glBindTexture(GL_TEXTURE_2D, [outputFramebuffer texture]); + glBindTexture(GL_TEXTURE_2D, [self->outputFramebuffer texture]); if (self.shouldSmoothlyScaleOutput) { glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_LINEAR_MIPMAP_LINEAR); @@ -322,18 +322,18 @@ - (BOOL)processImageWithCompletionHandler:(void (^)(void))completion; } runAsynchronouslyOnVideoProcessingQueue(^{ - for (id currentTarget in targets) + for (id currentTarget in self->targets) { - NSInteger indexOfObject = [targets indexOfObject:currentTarget]; - NSInteger textureIndexOfTarget = [[targetTextureIndices objectAtIndex:indexOfObject] integerValue]; + NSInteger indexOfObject = [self->targets indexOfObject:currentTarget]; + NSInteger textureIndexOfTarget = [[self->targetTextureIndices objectAtIndex:indexOfObject] integerValue]; [currentTarget setCurrentlyReceivingMonochromeInput:NO]; - [currentTarget setInputSize:pixelSizeOfImage atIndex:textureIndexOfTarget]; - [currentTarget setInputFramebuffer:outputFramebuffer atIndex:textureIndexOfTarget]; + [currentTarget setInputSize:self->pixelSizeOfImage atIndex:textureIndexOfTarget]; + [currentTarget setInputFramebuffer:self->outputFramebuffer atIndex:textureIndexOfTarget]; [currentTarget newFrameReadyAtTime:kCMTimeIndefinite atIndex:textureIndexOfTarget]; } - dispatch_semaphore_signal(imageUpdateSemaphore); + dispatch_semaphore_signal(self->imageUpdateSemaphore); if (completion != nil) { completion(); diff --git a/ScannerApp/Frameworks/GPUImage/framework/Source/iOS/GPUImageView.m b/ScannerApp/Frameworks/GPUImage/framework/Source/iOS/GPUImageView.m index e092b80..0871066 100755 --- a/ScannerApp/Frameworks/GPUImage/framework/Source/iOS/GPUImageView.m +++ b/ScannerApp/Frameworks/GPUImage/framework/Source/iOS/GPUImageView.m @@ -97,35 +97,35 @@ - (void)commonInit; runSynchronouslyOnVideoProcessingQueue(^{ [GPUImageContext useImageProcessingContext]; - displayProgram = [[GPUImageContext sharedImageProcessingContext] programForVertexShaderString:kGPUImageVertexShaderString fragmentShaderString:kGPUImagePassthroughFragmentShaderString]; - if (!displayProgram.initialized) + self->displayProgram = [[GPUImageContext sharedImageProcessingContext] programForVertexShaderString:kGPUImageVertexShaderString fragmentShaderString:kGPUImagePassthroughFragmentShaderString]; + if (!self->displayProgram.initialized) { - [displayProgram addAttribute:@"position"]; - [displayProgram addAttribute:@"inputTextureCoordinate"]; + [self->displayProgram addAttribute:@"position"]; + [self->displayProgram addAttribute:@"inputTextureCoordinate"]; - if (![displayProgram link]) + if (![self->displayProgram link]) { - NSString *progLog = [displayProgram programLog]; + NSString *progLog = [self->displayProgram programLog]; NSLog(@"Program link log: %@", progLog); - NSString *fragLog = [displayProgram fragmentShaderLog]; + NSString *fragLog = [self->displayProgram fragmentShaderLog]; NSLog(@"Fragment shader compile log: %@", fragLog); - NSString *vertLog = [displayProgram vertexShaderLog]; + NSString *vertLog = [self->displayProgram vertexShaderLog]; NSLog(@"Vertex shader compile log: %@", vertLog); - displayProgram = nil; + self->displayProgram = nil; NSAssert(NO, @"Filter shader link failed"); } } - displayPositionAttribute = [displayProgram attributeIndex:@"position"]; - displayTextureCoordinateAttribute = [displayProgram attributeIndex:@"inputTextureCoordinate"]; - displayInputTextureUniform = [displayProgram uniformIndex:@"inputImageTexture"]; // This does assume a name of "inputTexture" for the fragment shader + self->displayPositionAttribute = [self->displayProgram attributeIndex:@"position"]; + self->displayTextureCoordinateAttribute = [self->displayProgram attributeIndex:@"inputTextureCoordinate"]; + self->displayInputTextureUniform = [self->displayProgram uniformIndex:@"inputImageTexture"]; // This does assume a name of "inputTexture" for the fragment shader - [GPUImageContext setActiveShaderProgram:displayProgram]; - glEnableVertexAttribArray(displayPositionAttribute); - glEnableVertexAttribArray(displayTextureCoordinateAttribute); + [GPUImageContext setActiveShaderProgram:self->displayProgram]; + glEnableVertexAttribArray(self->displayPositionAttribute); + glEnableVertexAttribArray(self->displayTextureCoordinateAttribute); [self setBackgroundColorRed:0.0 green:0.0 blue:0.0 alpha:1.0]; - _fillMode = kGPUImageFillModePreserveAspectRatio; + self->_fillMode = kGPUImageFillModePreserveAspectRatio; [self createDisplayFramebuffer]; }); } @@ -240,9 +240,9 @@ - (void)recalculateViewGeometry; // CGFloat imageAspectRatio = inputImageSize.width / inputImageSize.height; // CGFloat viewAspectRatio = currentViewSize.width / currentViewSize.height; - CGRect insetRect = AVMakeRectWithAspectRatioInsideRect(inputImageSize, self.bounds); + CGRect insetRect = AVMakeRectWithAspectRatioInsideRect(self->inputImageSize, self.bounds); - switch(_fillMode) + switch(self->_fillMode) { case kGPUImageFillModeStretch: { @@ -262,14 +262,14 @@ - (void)recalculateViewGeometry; }; break; } - imageVertices[0] = -widthScaling; - imageVertices[1] = -heightScaling; - imageVertices[2] = widthScaling; - imageVertices[3] = -heightScaling; - imageVertices[4] = -widthScaling; - imageVertices[5] = heightScaling; - imageVertices[6] = widthScaling; - imageVertices[7] = heightScaling; + self->imageVertices[0] = -widthScaling; + self->imageVertices[1] = -heightScaling; + self->imageVertices[2] = widthScaling; + self->imageVertices[3] = -heightScaling; + self->imageVertices[4] = -widthScaling; + self->imageVertices[5] = heightScaling; + self->imageVertices[6] = widthScaling; + self->imageVertices[7] = heightScaling; }); // static const GLfloat imageVertices[] = { @@ -372,24 +372,24 @@ + (const GLfloat *)textureCoordinatesForRotation:(GPUImageRotationMode)rotationM - (void)newFrameReadyAtTime:(CMTime)frameTime atIndex:(NSInteger)textureIndex; { runSynchronouslyOnVideoProcessingQueue(^{ - [GPUImageContext setActiveShaderProgram:displayProgram]; + [GPUImageContext setActiveShaderProgram:self->displayProgram]; [self setDisplayFramebuffer]; - glClearColor(backgroundColorRed, backgroundColorGreen, backgroundColorBlue, backgroundColorAlpha); + glClearColor(self->backgroundColorRed, self->backgroundColorGreen, self->backgroundColorBlue, self->backgroundColorAlpha); glClear(GL_COLOR_BUFFER_BIT | GL_DEPTH_BUFFER_BIT); glActiveTexture(GL_TEXTURE4); - glBindTexture(GL_TEXTURE_2D, [inputFramebufferForDisplay texture]); - glUniform1i(displayInputTextureUniform, 4); + glBindTexture(GL_TEXTURE_2D, [self->inputFramebufferForDisplay texture]); + glUniform1i(self->displayInputTextureUniform, 4); - glVertexAttribPointer(displayPositionAttribute, 2, GL_FLOAT, 0, 0, imageVertices); - glVertexAttribPointer(displayTextureCoordinateAttribute, 2, GL_FLOAT, 0, 0, [GPUImageView textureCoordinatesForRotation:inputRotation]); + glVertexAttribPointer(self->displayPositionAttribute, 2, GL_FLOAT, 0, 0, self->imageVertices); + glVertexAttribPointer(self->displayTextureCoordinateAttribute, 2, GL_FLOAT, 0, 0, [GPUImageView textureCoordinatesForRotation:self->inputRotation]); glDrawArrays(GL_TRIANGLE_STRIP, 0, 4); [self presentFramebuffer]; - [inputFramebufferForDisplay unlock]; - inputFramebufferForDisplay = nil; + [self->inputFramebufferForDisplay unlock]; + self->inputFramebufferForDisplay = nil; }); } @@ -414,15 +414,15 @@ - (void)setInputSize:(CGSize)newSize atIndex:(NSInteger)textureIndex; runSynchronouslyOnVideoProcessingQueue(^{ CGSize rotatedSize = newSize; - if (GPUImageRotationSwapsWidthAndHeight(inputRotation)) + if (GPUImageRotationSwapsWidthAndHeight(self->inputRotation)) { rotatedSize.width = newSize.height; rotatedSize.height = newSize.width; } - if (!CGSizeEqualToSize(inputImageSize, rotatedSize)) + if (!CGSizeEqualToSize(self->inputImageSize, rotatedSize)) { - inputImageSize = rotatedSize; + self->inputImageSize = rotatedSize; [self recalculateViewGeometry]; } }); diff --git a/ScannerApp/Scanner.xcodeproj/project.pbxproj b/ScannerApp/Scanner.xcodeproj/project.pbxproj index 1b49ca7..54d36f6 100644 --- a/ScannerApp/Scanner.xcodeproj/project.pbxproj +++ b/ScannerApp/Scanner.xcodeproj/project.pbxproj @@ -136,7 +136,6 @@ 435307C41872825B00B04481 /* Default-Landscape@2x~ipad.png */ = {isa = PBXFileReference; lastKnownFileType = image.png; path = "Default-Landscape@2x~ipad.png"; sourceTree = ""; }; 435307C51872825B00B04481 /* Default@2x.png */ = {isa = PBXFileReference; lastKnownFileType = image.png; path = "Default@2x.png"; sourceTree = ""; }; 435307C61872825B00B04481 /* Default-Landscape~ipad.png */ = {isa = PBXFileReference; lastKnownFileType = image.png; path = "Default-Landscape~ipad.png"; sourceTree = ""; }; - 438C049918723EB10062E958 /* en */ = {isa = PBXFileReference; lastKnownFileType = file.xib; name = en; path = en.lproj/ViewController_iPad.xib; sourceTree = ""; }; 43BB38C8199D2A51009D7AB2 /* calibration.png */ = {isa = PBXFileReference; lastKnownFileType = image.png; path = calibration.png; sourceTree = ""; }; 6F03CB581862823000518C22 /* en */ = {isa = PBXFileReference; lastKnownFileType = text.plist.strings; name = en; path = InfoPlist.strings; sourceTree = ""; }; 6F03CB5E1862832600518C22 /* EAGLView.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = EAGLView.h; sourceTree = ""; }; @@ -152,6 +151,7 @@ 9591B5211E888E80005863DD /* Structure.framework */ = {isa = PBXFileReference; lastKnownFileType = wrapper.framework; name = Structure.framework; path = Frameworks/Structure.framework; sourceTree = ""; }; 9591B5231E888EA6005863DD /* GPUImage.xcodeproj */ = {isa = PBXFileReference; lastKnownFileType = "wrapper.pb-project"; name = GPUImage.xcodeproj; path = Frameworks/GPUImage/framework/GPUImage.xcodeproj; sourceTree = ""; }; 95F123031F44E60700265C89 /* Config.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; path = Config.h; sourceTree = ""; }; + A8426E672257C5A90024FEFE /* Base */ = {isa = PBXFileReference; lastKnownFileType = file.xib; name = Base; path = Base.lproj/ViewController_iPad.xib; sourceTree = ""; }; B60CC1BC1CFD5BB500EA51F4 /* toggleDepth.png */ = {isa = PBXFileReference; lastKnownFileType = image.png; path = toggleDepth.png; sourceTree = ""; }; B60CC1BE1CFE2FCD00EA51F4 /* toggleAWB.png */ = {isa = PBXFileReference; lastKnownFileType = image.png; path = toggleAWB.png; sourceTree = ""; }; B631B4521D6637F10064FB0D /* ViewController.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = ViewController.h; sourceTree = ""; }; @@ -367,7 +367,7 @@ 2AFC8D6C176F966000F7F3B5 /* Project object */ = { isa = PBXProject; attributes = { - LastUpgradeCheck = 0800; + LastUpgradeCheck = 1000; ORGANIZATIONNAME = Occipital; TargetAttributes = { 2AFC8D73176F966000F7F3B5 = { @@ -380,8 +380,10 @@ developmentRegion = English; hasScannedForEncodings = 0; knownRegions = ( + English, en, global, + Base, ); mainGroup = 2AFC8D6B176F966000F7F3B5; productRefGroup = 2AFC8D75176F966000F7F3B5 /* Products */; @@ -491,7 +493,7 @@ 438C049818723EB10062E958 /* ViewController_iPad.xib */ = { isa = PBXVariantGroup; children = ( - 438C049918723EB10062E958 /* en */, + A8426E672257C5A90024FEFE /* Base */, ); name = ViewController_iPad.xib; sourceTree = ""; @@ -515,12 +517,20 @@ CLANG_CXX_LANGUAGE_STANDARD = "gnu++0x"; CLANG_CXX_LIBRARY = "libc++"; CLANG_ENABLE_OBJC_ARC = YES; + CLANG_WARN_BLOCK_CAPTURE_AUTORELEASING = YES; CLANG_WARN_BOOL_CONVERSION = YES; + CLANG_WARN_COMMA = YES; CLANG_WARN_CONSTANT_CONVERSION = YES; + CLANG_WARN_DEPRECATED_OBJC_IMPLEMENTATIONS = YES; CLANG_WARN_EMPTY_BODY = YES; CLANG_WARN_ENUM_CONVERSION = YES; CLANG_WARN_INFINITE_RECURSION = YES; CLANG_WARN_INT_CONVERSION = YES; + CLANG_WARN_NON_LITERAL_NULL_CONVERSION = YES; + CLANG_WARN_OBJC_IMPLICIT_RETAIN_SELF = YES; + CLANG_WARN_OBJC_LITERAL_CONVERSION = YES; + CLANG_WARN_RANGE_LOOP_ANALYSIS = YES; + CLANG_WARN_STRICT_PROTOTYPES = YES; CLANG_WARN_SUSPICIOUS_MOVE = YES; CLANG_WARN_UNREACHABLE_CODE = YES; CLANG_WARN__DUPLICATE_METHOD_MATCH = YES; @@ -539,7 +549,7 @@ "$(inherited)", /Applications/Xcode.app/Contents/Developer/Toolchains/XcodeDefault.xctoolchain/usr/include, ); - IPHONEOS_DEPLOYMENT_TARGET = 9.3; + IPHONEOS_DEPLOYMENT_TARGET = 12.1; ONLY_ACTIVE_ARCH = YES; PRODUCT_BUNDLE_IDENTIFIER = "com.Scanner.*\n"; PROVISIONING_PROFILE = ""; @@ -562,7 +572,7 @@ GCC_PREFIX_HEADER = "Scanner/Scanner-Prefix.pch"; HEADER_SEARCH_PATHS = "./Frameworks/GPUImage/framework/**"; INFOPLIST_FILE = "Scanner/Scanner-Info.plist"; - IPHONEOS_DEPLOYMENT_TARGET = 9.3; + IPHONEOS_DEPLOYMENT_TARGET = 12.0; LD_RUNPATH_SEARCH_PATHS = "$(inherited) @executable_path/Frameworks"; OTHER_CFLAGS = "-DWRITE_LOCAL"; "OTHER_LDFLAGS[arch=*]" = "-ObjC"; @@ -580,12 +590,20 @@ CLANG_CXX_LANGUAGE_STANDARD = "gnu++0x"; CLANG_CXX_LIBRARY = "libc++"; CLANG_ENABLE_OBJC_ARC = YES; + CLANG_WARN_BLOCK_CAPTURE_AUTORELEASING = YES; CLANG_WARN_BOOL_CONVERSION = YES; + CLANG_WARN_COMMA = YES; CLANG_WARN_CONSTANT_CONVERSION = YES; + CLANG_WARN_DEPRECATED_OBJC_IMPLEMENTATIONS = YES; CLANG_WARN_EMPTY_BODY = YES; CLANG_WARN_ENUM_CONVERSION = YES; CLANG_WARN_INFINITE_RECURSION = YES; CLANG_WARN_INT_CONVERSION = YES; + CLANG_WARN_NON_LITERAL_NULL_CONVERSION = YES; + CLANG_WARN_OBJC_IMPLICIT_RETAIN_SELF = YES; + CLANG_WARN_OBJC_LITERAL_CONVERSION = YES; + CLANG_WARN_RANGE_LOOP_ANALYSIS = YES; + CLANG_WARN_STRICT_PROTOTYPES = YES; CLANG_WARN_SUSPICIOUS_MOVE = YES; CLANG_WARN_UNREACHABLE_CODE = YES; CLANG_WARN__DUPLICATE_METHOD_MATCH = YES; @@ -607,7 +625,7 @@ "$(inherited)", /Applications/Xcode.app/Contents/Developer/Toolchains/XcodeDefault.xctoolchain/usr/include, ); - IPHONEOS_DEPLOYMENT_TARGET = 9.3; + IPHONEOS_DEPLOYMENT_TARGET = 12.1; ONLY_ACTIVE_ARCH = YES; PRODUCT_BUNDLE_IDENTIFIER = "com.Scanner.*\n"; PROVISIONING_PROFILE = ""; @@ -629,7 +647,7 @@ GCC_PREFIX_HEADER = "Scanner/Scanner-Prefix.pch"; HEADER_SEARCH_PATHS = "./Frameworks/GPUImage/framework/**"; INFOPLIST_FILE = "Scanner/Scanner-Info.plist"; - IPHONEOS_DEPLOYMENT_TARGET = 9.3; + IPHONEOS_DEPLOYMENT_TARGET = 12.0; LD_RUNPATH_SEARCH_PATHS = "$(inherited) @executable_path/Frameworks"; OTHER_CFLAGS = "-DWRITE_LOCAL"; "OTHER_LDFLAGS[arch=*]" = "-ObjC"; diff --git a/ScannerApp/Scanner/en.lproj/ViewController_iPad.xib b/ScannerApp/Scanner/Base.lproj/ViewController_iPad.xib similarity index 100% rename from ScannerApp/Scanner/en.lproj/ViewController_iPad.xib rename to ScannerApp/Scanner/Base.lproj/ViewController_iPad.xib diff --git a/ScannerApp/Scanner/Scanner-Info.plist b/ScannerApp/Scanner/Scanner-Info.plist index 1dac8f5..17ccb29 100644 --- a/ScannerApp/Scanner/Scanner-Info.plist +++ b/ScannerApp/Scanner/Scanner-Info.plist @@ -21,11 +21,11 @@ CFBundlePackageType APPL CFBundleShortVersionString - 1.4.2 + 1.4.3 CFBundleSignature ???? CFBundleVersion - 20170930211932 + 20190409143822 LSRequiresIPhoneOS NSAppTransportSecurity diff --git a/SensReader/README.md b/SensReader/README.md index 1a9c5c2..b6835b1 100644 --- a/SensReader/README.md +++ b/SensReader/README.md @@ -1,10 +1,11 @@ # Sens File Format (see [sensorData.h](c++/src/sensorData.h)): +``` struct SensorData { unsigned int m_versionNumber; - std::string m_sensorName; + std::string m_sensorName; - CalibrationData m_calibrationColor; //4x4 intrinsic matrix - CalibrationData m_calibrationDepth; //4x4 intrinsic matrix + CalibrationData m_calibrationColor; //4x4 intrinsic matrix + CalibrationData m_calibrationDepth; //4x4 intrinsic matrix COMPRESSION_TYPE_COLOR m_colorCompressionType; //scannet color frames are typically jpeg encoded COMPRESSION_TYPE_DEPTH m_depthCompressionType; //scannet depth frames are typically zLib encoded @@ -13,14 +14,26 @@ struct SensorData { unsigned int m_colorHeight; unsigned int m_depthWidth; unsigned int m_depthHeight; - float m_depthShift; //conversion from float[m] to ushort (typically 1000.0f) + float m_depthShift; //conversion from float[m] to ushort (typically 1000.0f) - std::vector m_frames; // <= Main data (contains timestamps and rigid transforms) + std::vector m_frames; // <= Main data (contains timestamps and rigid transforms) std::vector m_IMUFrames; } +// example get color frame, depth frame, pose (camera to world) +SensorData sd("sensfile.sens"); +unsigned int frame = 0; +vec3uc* color = sd.decompressColorAlloc(frame); // note: must be freed after use +unsigned short* depth = sd.decompressDepthAlloc(frame); // note: must be freed after use +const mat4f cameraToWorld = sd.m_frames[frame].getCameraToWorld(); +std::free(color); +std::free(depth); +// if using mlib, can save a frame to a point cloud with: +sd.saveToPointCloud("frame" + std::to_string(frame) + ".ply", frame); +``` + ## C++ ToolKit See [c++/](c++) ## Simple Python Data Exporter -See [python/](python) \ No newline at end of file +See [python/](python) diff --git a/SensReader/c++/README.txt b/SensReader/c++/README.txt index 246287c..906db83 100644 --- a/SensReader/c++/README.txt +++ b/SensReader/c++/README.txt @@ -11,6 +11,8 @@ Run: Hint: keep the sens files as they are a nice represention see processFrame(..) to decode independent frames + +For additional functionality (vector/matrix/point cloud classes, etc), include the mLib library: https://github.com/niessner/mLib. - tested under Windows10 VS2013 - tested 14.04.1-Ubuntu: g++ and clang diff --git a/SensReader/python/README.md b/SensReader/python/README.md index 7ab28cd..60da5a8 100644 --- a/SensReader/python/README.md +++ b/SensReader/python/README.md @@ -1,5 +1,7 @@ # Data Exporter +Developed and tested with python 2.7. + Usage: ``` python reader.py --filename [.sens file to export data from] --output_path [output directory to export data to] diff --git a/SensReader/python/SensorData.py b/SensReader/python/SensorData.py index 95c8d99..16215ca 100644 --- a/SensReader/python/SensorData.py +++ b/SensReader/python/SensorData.py @@ -3,6 +3,8 @@ import numpy as np import zlib import imageio +import cv2 +import png COMPRESSION_TYPE_COLOR = {-1:'unknown', 0:'raw', 1:'png', 2:'jpeg'} COMPRESSION_TYPE_DEPTH = {-1:'unknown', 0:'raw_ushort', 1:'zlib_ushort', 2:'occi_ushort'} @@ -73,22 +75,29 @@ def load(self, filename): self.frames.append(frame) - def export_depth_images(self, output_path): + def export_depth_images(self, output_path, image_size=None, frame_skip=1): if not os.path.exists(output_path): os.makedirs(output_path) - print 'exporting', len(self.frames), ' depth frames to', output_path - for f in range(len(self.frames)): + print 'exporting', len(self.frames)//frame_skip, ' depth frames to', output_path + for f in range(0, len(self.frames), frame_skip): depth_data = self.frames[f].decompress_depth(self.depth_compression_type) depth = np.fromstring(depth_data, dtype=np.uint16).reshape(self.depth_height, self.depth_width) - imageio.imwrite(os.path.join(output_path, str(f) + '.png'), depth) - - - def export_color_images(self, output_path): + if image_size is not None: + depth = cv2.resize(depth, (image_size[1], image_size[0]), interpolation=cv2.INTER_NEAREST) + #imageio.imwrite(os.path.join(output_path, str(f) + '.png'), depth) + with open(os.path.join(output_path, str(f) + '.png'), 'wb') as f: # write 16-bit + writer = png.Writer(width=depth.shape[1], height=depth.shape[0], bitdepth=16) + depth = depth.reshape(-1, depth.shape[1]).tolist() + writer.write(f, depth) + + def export_color_images(self, output_path, image_size=None, frame_skip=1): if not os.path.exists(output_path): os.makedirs(output_path) - print 'exporting', len(self.frames), 'color frames to', output_path - for f in range(len(self.frames)): + print 'exporting', len(self.frames)//frame_skip, 'color frames to', output_path + for f in range(0, len(self.frames), frame_skip): color = self.frames[f].decompress_color(self.color_compression_type) + if image_size is not None: + color = cv2.resize(color, (image_size[1], image_size[0]), interpolation=cv2.INTER_NEAREST) imageio.imwrite(os.path.join(output_path, str(f) + '.jpg'), color) @@ -98,11 +107,11 @@ def save_mat_to_file(self, matrix, filename): np.savetxt(f, line[np.newaxis], fmt='%f') - def export_poses(self, output_path): + def export_poses(self, output_path, frame_skip=1): if not os.path.exists(output_path): os.makedirs(output_path) - print 'exporting', len(self.frames), 'camera poses to', output_path - for f in range(len(self.frames)): + print 'exporting', len(self.frames)//frame_skip, 'camera poses to', output_path + for f in range(0, len(self.frames), frame_skip): self.save_mat_to_file(self.frames[f].camera_to_world, os.path.join(output_path, str(f) + '.txt')) @@ -114,4 +123,3 @@ def export_intrinsics(self, output_path): self.save_mat_to_file(self.extrinsic_color, os.path.join(output_path, 'extrinsic_color.txt')) self.save_mat_to_file(self.intrinsic_depth, os.path.join(output_path, 'intrinsic_depth.txt')) self.save_mat_to_file(self.extrinsic_depth, os.path.join(output_path, 'extrinsic_depth.txt')) - diff --git a/Server/tools/meshclean/clean.mlx b/Server/tools/meshclean/clean.mlx new file mode 100644 index 0000000..55f9fe2 --- /dev/null +++ b/Server/tools/meshclean/clean.mlx @@ -0,0 +1,11 @@ + + + + + + + + + + + diff --git a/Server/tools/meshclean/cleanLoRes.mlx b/Server/tools/meshclean/cleanLoRes.mlx new file mode 100644 index 0000000..e5c177a --- /dev/null +++ b/Server/tools/meshclean/cleanLoRes.mlx @@ -0,0 +1,11 @@ + + + + + + + + + + + diff --git a/Server/tools/meshclean/simplify.mlx b/Server/tools/meshclean/simplify.mlx new file mode 100644 index 0000000..f48fe2c --- /dev/null +++ b/Server/tools/meshclean/simplify.mlx @@ -0,0 +1,25 @@ + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/Server/tools/recons/zParametersBundlingScanNet.txt b/Server/tools/recons/zParametersBundlingScanNet.txt new file mode 100644 index 0000000..e4d5625 --- /dev/null +++ b/Server/tools/recons/zParametersBundlingScanNet.txt @@ -0,0 +1,76 @@ + + +s_verbose = true; // output debug statements (invalid frames etc) + +s_erodeSIFTdepth = true; +s_sendUplinkFeedbackImage = true; + +s_recordSolverConvergence = false; + +s_enablePerFrameTimings = true; +s_enableGlobalTimings = false; + +s_widthSIFT = 640; +s_heightSIFT = 480; + +s_minKeyScale = 3.0f;//5.0f; +s_siftMatchThresh = 0.7f;//0.5f; +s_siftMatchRatioMaxLocal = 0.8f; +s_siftMatchRatioMaxGlobal = 0.8f; + +s_optMaxResThresh = 0.12f; //not squared (per axis component) +s_denseDistThresh = 0.15f; +s_denseNormalThresh = 0.95f; +s_denseColorThresh = 0.1f; +s_denseColorGradientMin = 0.005f; +s_denseDepthMin = 0.5f; +s_denseDepthMax = 4.0f;//4.5f; +s_denseOverlapCheckSubsampleFactor = 4; + +s_maxNumImages = 2000; +s_submapSize = 10; +s_maxNumKeysPerImage = 512; +s_maxNumCorrPerImage = 2000; // over-conservative +s_maxNumResidualsRemoved = 10; + +s_useLocalDense = true; +s_numOptPerResidualRemoval = 1; + +s_numLocalNonLinIterations = 2; +s_numLocalLinIterations = 100; +s_numGlobalNonLinIterations = 3; +s_numGlobalLinIterations = 150; + +s_downsampledWidth = 80; +s_downsampledHeight = 60; + + +//dense term filtering +s_colorDownSigma = 2.5f; +s_depthDownSigmaD = 1.0f; +s_depthDownSigmaR = 0.05f; + +s_maxKabschResidual2 = 0.0006f; + +s_projCorrDistThres = 0.15f; +s_projCorrNormalThres = 0.95f; +s_projCorrColorThresh = 0.1f; + +s_verifySiftErrThresh = 0.12f; +s_verifySiftCorrThresh = 0.02f; + +s_useLocalVerify = true; +s_verifyOptErrThresh = 0.05f; +s_verifyOptCorrThresh = 0.001f; + +s_surfAreaPcaThresh = 0.032f; + +s_minNumMatchesLocal = 4; +s_minNumMatchesGlobal = 4; + +// filtering +s_depthSigmaD = 2.0f; //bilateral filter sigma domain +s_depthSigmaR = 0.05f; //bilateral filter sigma range +s_depthFilter = true; //bilateral filter enabled depth + +s_useComprehensiveFrameInvalidation = true; \ No newline at end of file diff --git a/Server/tools/recons/zParametersScanNet.txt b/Server/tools/recons/zParametersScanNet.txt new file mode 100644 index 0000000..45a8649 --- /dev/null +++ b/Server/tools/recons/zParametersScanNet.txt @@ -0,0 +1,126 @@ +// 0=Kinect; 1=PrimeSense; 2=KinectOne; 3=BinaryDumpReader; 4=NetworkSensor; 5=IntelSensor; 6=RealSense; 7=StructureSensor; 8=SensorDataReader +s_sensorIdx = 8; + +s_numSolveFramesBeforeExit = 60; //#frames to run after scan is done before saving ply and exiting; -1 to keep running +s_overwriteOrigSensTrajectory = true; + +s_generateVideo = false; +s_generateVideoDir = "output/"; +s_printTimingsDirectory = ""; +s_printConvergenceFile = ""; +s_topVideoTransformWorld = 1.0f 0.0f 0.0f 0.0f 0.0f 1.0f 0.0f 0.0f 0.0f 0.0f 1.0f 0.0f 0.0f 0.0f 0.0f 1.0f; +s_topVideoCameraPose = 0.0f 0.0f 0.0f 0.0f; //rotation (deg around z axis), translation (m) +s_topVideoMinMax = 0.0f 0.0f; + +s_windowWidth = 640; //render window width +s_windowHeight = 480; //render window height +//s_windowWidth = 1600; //render window width +//s_windowHeight = 1200; //render window height + +s_integrationWidth = 320; //input depth gets re-sampled to this width (decrease to improve perf.) +s_integrationHeight = 240; //input depth gets re-sampled to this height (decrease to improve perf.) +s_rayCastWidth = 320; //should be same as integration except if rendering video +s_rayCastHeight = 240; + +//trajectory manager/reintegration +s_maxFrameFixes = 30; //max number of frames reintegrated per frame +s_topNActive = 30; //max number of active elements to be reintegrated (sorted list) +s_minPoseDistSqrt = 0.0f; //reintegrate everything above that pose distance (squared dist) + +//////////////////////////////////// +// **** DEPTH SENSING BELOW ***** // +//////////////////////////////////// + +s_sensorDepthMax = 6.0f; //maximum sensor depth in meter +s_sensorDepthMin = 0.1f; //minimum sensor depth in meter +s_renderDepthMax = 6.0f; //maximum render depth in meter +s_renderDepthMin = 0.1f; //minimum render depth in meter + +// cropping options for PrimeSense & Asus Xtion Pro +s_enableColorCropping = false; +s_colorCropX = 320; +s_colorCropY = 272; +s_colorCropWidth = 640; +s_colorCropHeight = 480; + +// sdf and hash settings +s_SDFVoxelSize = 0.010f; //voxel size in meter (IMPORTANT: reduces to improve perf.) +s_SDFMarchingCubeThreshFactor = 10.0f; //marching cube thresh: s_SDFMarchingCubeThresh = s_SDFMarchingCubeThreshFactor*s_SDFVoxelSize +s_SDFTruncation = 0.06f; //truncation in meter +s_SDFTruncationScale = 0.02f; //truncation scale in meter per meter +s_SDFMaxIntegrationDistance = 4.0f; //maximum integration in meter +s_SDFIntegrationWeightSample = 1; //weight for an integrated depth value +s_SDFIntegrationWeightMax = 99999999; //maximum integration weight for a voxel +// s_SDFBlockSize is pound defined (SDF_BLOCK_SIZE) +// s_hashBucketSize is pound defined (HASH_BUCKET_SIZE) +s_hashNumBuckets = 800000; //smaller voxels require more space +s_hashNumSDFBlocks = 600000;//100000; //smaller voxels require more space +s_hashMaxCollisionLinkedListSize = 7; + +// raycast +s_SDFRayIncrementFactor = 0.8f; //(don't touch) s_SDFRayIncrement = s_SDFRayIncrementFactor*s_SDFTrunaction; +s_SDFRayThresSampleDistFactor = 50.5f; //(don't touch) s_SDFRayThresSampleDist = s_SDFRayThresSampleDistFactor*s_rayIncrement; +s_SDFRayThresDistFactor = 50.0f; //(don't touch) s_SDFRayThresDist = s_SDFRayThresSampleDistFactor*s_rayIncrement; +s_SDFUseGradients = false; //analytic gradients for rendering + + +s_binaryDumpSensorFile = ""; +s_binaryDumpSensorUseTrajectory = false; + +// filtering (unused in FriedLiver) +s_depthSigmaD = 2.0f; //bilateral filter sigma domain +s_depthSigmaR = 0.1f; //bilateral filter sigma range +s_depthFilter = false; //bilateral filter enabled depth + +s_colorSigmaD = 2.0f; //bilateral filter sigma domain +s_colorSigmaR = 0.1f; //bilateral filter sigma range +s_colorFilter = false; //bilateral filter enabled depth + +s_integrationEnabled = true; +s_trackingEnabled = true; +s_timingsDetailledEnabled = true; //enable timing output +s_timingsTotalEnabled = false; //enable timing output +s_garbageCollectionEnabled = true; +s_garbageCollectionStarve = 0; //decrement the voxel weight every n'th frame + +// rendering +s_materialShininess = 16.0f; +s_materialAmbient = 0.75f 0.65f 0.5f 1.0f; +s_materialDiffuse = 1.0f 0.9f 0.7f 1.0f; +s_materialSpecular = 1.0f 1.0f 1.0f 1.0f; +s_lightAmbient = 0.4f 0.4f 0.4f 1.0f; +s_lightDiffuse = 0.6f 0.52944f 0.4566f 0.6f; +s_lightSpecular = 0.3f 0.3f 0.3f 1.0f; +s_lightDirection = 0.0f -1.0f 2.0f; + +s_RenderMode = 1; +s_playData = true; + +s_renderingDepthDiscontinuityThresOffset = 0.012f; // discontinuity offset in meter +s_renderingDepthDiscontinuityThresLin = 0.001f; // additional discontinuity threshold per meter +s_remappingDepthDiscontinuityThresOffset = 0.012f; // discontinuity offset in meter +s_remappingDepthDiscontinuityThresLin = 0.01f; + +s_bUseCameraCalibration = false; + +s_marchingCubesMaxNumTriangles = 5000000; // max buffer size for marching cube + +//streaming parameters +s_streamingEnabled = false; +s_streamingVoxelExtents = 1.0f 1.0f 1.0f; +s_streamingGridDimensions = 257 257 257; // dimensions have to be odd (number of samples) +s_streamingMinGridPos = -128 -128 -128; +s_streamingInitialChunkListSize = 2000; +s_streamingRadius = 5.0f; // Depends on DepthMin and DepthMax +s_streamingPos = 0.0f 0.0f 3.0f 1.0f; // Depends on DepthMin and DepthMax +s_streamingOutParts = 80; // number of frames required to sweep through the entire hash + +//recording of the input data +s_recordData = false; // master flag for data recording: enables or disables data recording +s_recordCompression = true; //if recoding is enabled; then compression is used (.sens instead of .sensor) +s_recordDataWidth = 640; //only applies to the non compressed version (see RGBDSensor.cpp) +s_recordDataHeight = 480; //only applies to the non compressed version (see RGBDSensor.cpp) +s_recordDataFile = "dump/recording.sens"; +s_reconstructionEnabled = true; + + diff --git a/Tasks/Benchmark/scannet_test.txt b/Tasks/Benchmark/scannetv1_test.txt similarity index 100% rename from Tasks/Benchmark/scannet_test.txt rename to Tasks/Benchmark/scannetv1_test.txt diff --git a/Tasks/Benchmark/scannet_train.txt b/Tasks/Benchmark/scannetv1_train.txt similarity index 100% rename from Tasks/Benchmark/scannet_train.txt rename to Tasks/Benchmark/scannetv1_train.txt diff --git a/Tasks/Benchmark/scannet_val.txt b/Tasks/Benchmark/scannetv1_val.txt similarity index 100% rename from Tasks/Benchmark/scannet_val.txt rename to Tasks/Benchmark/scannetv1_val.txt diff --git a/Tasks/Benchmark/scannet_rob_test.txt b/Tasks/Benchmark/scannetv2_test.txt similarity index 80% rename from Tasks/Benchmark/scannet_rob_test.txt rename to Tasks/Benchmark/scannetv2_test.txt index c04e670..79d15b0 100644 --- a/Tasks/Benchmark/scannet_rob_test.txt +++ b/Tasks/Benchmark/scannetv2_test.txt @@ -78,3 +78,23 @@ scene0783_00 scene0784_00 scene0785_00 scene0786_00 +scene0787_00 +scene0788_00 +scene0789_00 +scene0790_00 +scene0791_00 +scene0792_00 +scene0793_00 +scene0794_00 +scene0795_00 +scene0796_00 +scene0797_00 +scene0798_00 +scene0799_00 +scene0800_00 +scene0801_00 +scene0802_00 +scene0803_00 +scene0804_00 +scene0805_00 +scene0806_00 diff --git a/Tasks/Benchmark/scannet_trainval.txt b/Tasks/Benchmark/scannetv2_train.txt similarity index 100% rename from Tasks/Benchmark/scannet_trainval.txt rename to Tasks/Benchmark/scannetv2_train.txt diff --git a/Tasks/Benchmark/scannetv2_val.txt b/Tasks/Benchmark/scannetv2_val.txt new file mode 100644 index 0000000..b9e7d92 --- /dev/null +++ b/Tasks/Benchmark/scannetv2_val.txt @@ -0,0 +1,312 @@ +scene0568_00 +scene0568_01 +scene0568_02 +scene0304_00 +scene0488_00 +scene0488_01 +scene0412_00 +scene0412_01 +scene0217_00 +scene0019_00 +scene0019_01 +scene0414_00 +scene0575_00 +scene0575_01 +scene0575_02 +scene0426_00 +scene0426_01 +scene0426_02 +scene0426_03 +scene0549_00 +scene0549_01 +scene0578_00 +scene0578_01 +scene0578_02 +scene0665_00 +scene0665_01 +scene0050_00 +scene0050_01 +scene0050_02 +scene0257_00 +scene0025_00 +scene0025_01 +scene0025_02 +scene0583_00 +scene0583_01 +scene0583_02 +scene0701_00 +scene0701_01 +scene0701_02 +scene0580_00 +scene0580_01 +scene0565_00 +scene0169_00 +scene0169_01 +scene0655_00 +scene0655_01 +scene0655_02 +scene0063_00 +scene0221_00 +scene0221_01 +scene0591_00 +scene0591_01 +scene0591_02 +scene0678_00 +scene0678_01 +scene0678_02 +scene0462_00 +scene0427_00 +scene0595_00 +scene0193_00 +scene0193_01 +scene0164_00 +scene0164_01 +scene0164_02 +scene0164_03 +scene0598_00 +scene0598_01 +scene0598_02 +scene0599_00 +scene0599_01 +scene0599_02 +scene0328_00 +scene0300_00 +scene0300_01 +scene0354_00 +scene0458_00 +scene0458_01 +scene0423_00 +scene0423_01 +scene0423_02 +scene0307_00 +scene0307_01 +scene0307_02 +scene0606_00 +scene0606_01 +scene0606_02 +scene0432_00 +scene0432_01 +scene0608_00 +scene0608_01 +scene0608_02 +scene0651_00 +scene0651_01 +scene0651_02 +scene0430_00 +scene0430_01 +scene0689_00 +scene0357_00 +scene0357_01 +scene0574_00 +scene0574_01 +scene0574_02 +scene0329_00 +scene0329_01 +scene0329_02 +scene0153_00 +scene0153_01 +scene0616_00 +scene0616_01 +scene0671_00 +scene0671_01 +scene0618_00 +scene0382_00 +scene0382_01 +scene0490_00 +scene0621_00 +scene0607_00 +scene0607_01 +scene0149_00 +scene0695_00 +scene0695_01 +scene0695_02 +scene0695_03 +scene0389_00 +scene0377_00 +scene0377_01 +scene0377_02 +scene0342_00 +scene0139_00 +scene0629_00 +scene0629_01 +scene0629_02 +scene0496_00 +scene0633_00 +scene0633_01 +scene0518_00 +scene0652_00 +scene0406_00 +scene0406_01 +scene0406_02 +scene0144_00 +scene0144_01 +scene0494_00 +scene0278_00 +scene0278_01 +scene0316_00 +scene0609_00 +scene0609_01 +scene0609_02 +scene0609_03 +scene0084_00 +scene0084_01 +scene0084_02 +scene0696_00 +scene0696_01 +scene0696_02 +scene0351_00 +scene0351_01 +scene0643_00 +scene0644_00 +scene0645_00 +scene0645_01 +scene0645_02 +scene0081_00 +scene0081_01 +scene0081_02 +scene0647_00 +scene0647_01 +scene0535_00 +scene0353_00 +scene0353_01 +scene0353_02 +scene0559_00 +scene0559_01 +scene0559_02 +scene0593_00 +scene0593_01 +scene0246_00 +scene0653_00 +scene0653_01 +scene0064_00 +scene0064_01 +scene0356_00 +scene0356_01 +scene0356_02 +scene0030_00 +scene0030_01 +scene0030_02 +scene0222_00 +scene0222_01 +scene0338_00 +scene0338_01 +scene0338_02 +scene0378_00 +scene0378_01 +scene0378_02 +scene0660_00 +scene0553_00 +scene0553_01 +scene0553_02 +scene0527_00 +scene0663_00 +scene0663_01 +scene0663_02 +scene0664_00 +scene0664_01 +scene0664_02 +scene0334_00 +scene0334_01 +scene0334_02 +scene0046_00 +scene0046_01 +scene0046_02 +scene0203_00 +scene0203_01 +scene0203_02 +scene0088_00 +scene0088_01 +scene0088_02 +scene0088_03 +scene0086_00 +scene0086_01 +scene0086_02 +scene0670_00 +scene0670_01 +scene0256_00 +scene0256_01 +scene0256_02 +scene0249_00 +scene0441_00 +scene0658_00 +scene0704_00 +scene0704_01 +scene0187_00 +scene0187_01 +scene0131_00 +scene0131_01 +scene0131_02 +scene0207_00 +scene0207_01 +scene0207_02 +scene0461_00 +scene0011_00 +scene0011_01 +scene0343_00 +scene0251_00 +scene0077_00 +scene0077_01 +scene0684_00 +scene0684_01 +scene0550_00 +scene0686_00 +scene0686_01 +scene0686_02 +scene0208_00 +scene0500_00 +scene0500_01 +scene0552_00 +scene0552_01 +scene0648_00 +scene0648_01 +scene0435_00 +scene0435_01 +scene0435_02 +scene0435_03 +scene0690_00 +scene0690_01 +scene0693_00 +scene0693_01 +scene0693_02 +scene0700_00 +scene0700_01 +scene0700_02 +scene0699_00 +scene0231_00 +scene0231_01 +scene0231_02 +scene0697_00 +scene0697_01 +scene0697_02 +scene0697_03 +scene0474_00 +scene0474_01 +scene0474_02 +scene0474_03 +scene0474_04 +scene0474_05 +scene0355_00 +scene0355_01 +scene0146_00 +scene0146_01 +scene0146_02 +scene0196_00 +scene0702_00 +scene0702_01 +scene0702_02 +scene0314_00 +scene0277_00 +scene0277_01 +scene0277_02 +scene0095_00 +scene0095_01 +scene0015_00 +scene0100_00 +scene0100_01 +scene0100_02 +scene0558_00 +scene0558_01 +scene0558_02 +scene0685_00 +scene0685_01 +scene0685_02 diff --git a/img/legend.png b/img/legend.png index d14e19c..0354c93 100644 Binary files a/img/legend.png and b/img/legend.png differ