Merge pull request #2696 from erwincoumans/master

pybullet: cache zipfile, and also allow to read zipfile from memory (and already registered fileIO plugins)

examples/SharedMemory/plugins/fileIOPlugin/fileIOPlugin.cpp

@@ -72,6 +72,7 @@ struct InMemoryFileIO : public CommonFileIOInterface
 	{
 		for (int i=0;i<m_fileCache.size();i++)
 		{
+			b3HashString name = m_fileCache.getKeyAtIndex(i);
 			InMemoryFile** memPtr = m_fileCache.getAtIndex(i);
 			if (memPtr && *memPtr)
 			{
@@ -80,6 +81,7 @@ struct InMemoryFileIO : public CommonFileIOInterface
 				m_numFrees++;
 				delete (mem);
 				m_numFrees++;
+				m_fileCache.remove(name);
 			}
 		}
 	}
@@ -319,6 +321,7 @@ struct WrapperFileIO : public CommonFileIOInterface
 		{
 			removeFileIOInterface(i);
 		}
+		m_cachedFiles.clearCache();
 	}
 
 	int addFileIOInterface(CommonFileIOInterface* fileIO)

examples/SharedMemory/plugins/fileIOPlugin/zipFileIO.h

@@ -9,19 +9,51 @@ struct ZipFileIO : public CommonFileIOInterface
 
 	unzFile	m_fileHandles[B3_ZIP_FILEIO_MAX_FILES ];
 	int m_numFileHandles;
-
+	unzFile m_zipfile;
+	voidpf m_stream;
+	unz_global_info m_global_info;
+	bool m_memoryFile;
+	b3AlignedObjectArray<char> m_buffer;
+	
 	ZipFileIO(int fileIOType, const char* zipfileName, CommonFileIOInterface* wrapperFileIO)
 		:CommonFileIOInterface(fileIOType,0),
 		m_zipfileName(zipfileName),
-		m_numFileHandles(0)
+		m_numFileHandles(0),
+		m_stream(0),
+		m_memoryFile(false)
 	{
 		m_pathPrefix = m_zipfileName.c_str();
 		for (int i=0;i<B3_ZIP_FILEIO_MAX_FILES ;i++)
 		{
 			m_fileHandles[i]=0;
 		}
+		m_zipfile = unzOpen(m_zipfileName.c_str());
+
+		if (m_zipfile == 0)
+		{
+			int fileIO = wrapperFileIO->fileOpen(m_zipfileName.c_str(), "rb");
+			if (fileIO >= 0)
+			{
+				int stream_size = wrapperFileIO->getFileSize(fileIO);
+				m_buffer.resize(stream_size);
+				int read_bytes = wrapperFileIO->fileRead(fileIO, &m_buffer[0], stream_size);
+				b3Assert(read_bytes == stream_size);
+				if (read_bytes != stream_size)
+				{
+					printf("Error: mismatch reading file %s, expected %d bytes, read %d\n", m_zipfileName.c_str(), stream_size, read_bytes);
+				}
+				zlib_filefunc_def api;  // callbacks for in-mem file
+				
+				m_stream = mem_simple_create_file(&api, &m_buffer[0], stream_size);
+
+				m_zipfile = unzAttach(m_stream, &api);
+				m_memoryFile = true;
+				wrapperFileIO->fileClose(fileIO);
+			}
+		}
 	}
 
+	
 		
 	static bool FileIOPluginFindFile(void* userPtr, const char* orgFileName, char* relativeFileName, int maxRelativeFileNameMaxLen)
 	{
@@ -29,14 +61,36 @@ struct ZipFileIO : public CommonFileIOInterface
 		return fileIo->findFile(orgFileName, relativeFileName, maxRelativeFileNameMaxLen);
 	}
 
+	void closeZipFile()
+	{
+		if (m_zipfile)
+		{
+			if (m_memoryFile)
+			{
+				unzDetach(&m_zipfile);
+			}
+			else
+			{
+				unzClose(m_zipfile);
+			}
+		}
+		m_zipfile = 0;
+	}
+
 	virtual ~ZipFileIO()
 	{
 		for (int i=0;i<B3_ZIP_FILEIO_MAX_FILES;i++)
 		{
 			fileClose(i);
 		}
+		closeZipFile();
+		if (m_stream)
+		{
+			mem_simple_destroy_file(m_stream);
+		}
 	}
 
+	
 	virtual int fileOpen(const char* fileName, const char* mode)
 	{
 		
@@ -52,10 +106,8 @@ struct ZipFileIO : public CommonFileIOInterface
 		}
 		if (slot>=0)
 		{
-			unzFile zipfile;
-			unz_global_info m_global_info;
-			zipfile = unzOpen(m_zipfileName.c_str());
-			if (zipfile == NULL)
+			
+			if (m_zipfile == NULL)
 			{
 				printf("%s: not found\n", m_zipfileName.c_str());
 				slot = -1;
@@ -63,48 +115,41 @@ struct ZipFileIO : public CommonFileIOInterface
 			{
 				
 				int result = 0;
-				result = unzGetGlobalInfo(zipfile, &m_global_info );
+				result = unzGetGlobalInfo(m_zipfile, &m_global_info );
 				if (result != UNZ_OK)
 				{
 					printf("could not read file global info from %s\n", m_zipfileName.c_str());
-					unzClose(zipfile);
-					zipfile = 0;
 					slot = -1;
 				}
 			}
 			if (slot >=0)
 			{
-				int result = unzLocateFile(zipfile, fileName, 0);
+				int result = unzLocateFile(m_zipfile, fileName, 0);
 				if (result == UNZ_OK)
 				{
 					unz_file_info info;
-					result = unzGetCurrentFileInfo(zipfile, &info, NULL, 0, NULL, 0, NULL, 0);
+					result = unzGetCurrentFileInfo(m_zipfile, &info, NULL, 0, NULL, 0, NULL, 0);
 					if (result != UNZ_OK)
 					{
 						printf("unzGetCurrentFileInfo() != UNZ_OK (%d)\n", result);
 						slot=-1;
-						unzClose(zipfile);
-						zipfile = 0;
 					}
 					else
 					{
-						result = unzOpenCurrentFile(zipfile);
+						result = unzOpenCurrentFile(m_zipfile);
 						if (result == UNZ_OK)
 						{
 							printf("zipFile::fileOpen %s in mode %s in fileHandle %d\n", fileName, mode, slot);
-							m_fileHandles[slot] = zipfile;
+							m_fileHandles[slot] = m_zipfile;
 						} else
 						{
 							slot=-1;
-							unzClose(zipfile);
-							zipfile = 0;
 						}
 					}
 				} else
 				{
 					slot=-1;
-					unzClose(zipfile);
-					zipfile = 0;
+					
 				}
 			}
 		}
@@ -147,7 +192,6 @@ struct ZipFileIO : public CommonFileIOInterface
 			if (f)
 			{
 				printf("zipFile::fileClose slot %d\n", fileHandle);
-				unzClose(f);
 				m_fileHandles[fileHandle]=0;
 			}
 		}

examples/ThirdPartyLibs/minizip/ioapi.c

@@ -232,3 +232,250 @@ void fill_fopen64_filefunc(zlib_filefunc64_def* pzlib_filefunc_def)
 	pzlib_filefunc_def->zerror_file = ferror_file_func;
 	pzlib_filefunc_def->opaque = NULL;
 }
+
+/* This file is from the proposed iomem_simple package found at
+   http://code.trak.dk/
+   Local modifications exist to fix various security bugs. See this
+   file's revision history.
+*/
+
+/* ioapi_mem2.c -- IO base function header for compress/uncompress .zip
+   files using zlib + zip or unzip API
+
+   This version of ioapi is designed to access memory rather than files.
+   We do use a region of memory to put data in to and take it out of. We do
+   not have auto-extending buffers and do not inform anyone else that the
+   data has been written. It is really intended for accessing a zip archive
+   embedded in an application such that I can write an installer with no
+   external files. Creation of archives has not been attempted, although
+   parts of the framework are present.
+
+   Based on Unzip ioapi.c version 0.22, May 19th, 2003
+
+   Copyright (C) 1998-2003 Gilles Vollant
+             (C) 2003 Justin Fletcher
+
+   Dynamically allocated memory version. Troels K 2004
+      mem_close deletes the data: file is single-session. No filenames.
+
+   This file is under the same license as the Unzip tool it is distributed
+   with.
+*/
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+
+#include "unzip.h"
+
+
+#ifndef ZOFF_T
+#define ZOFF_T uLong /* bw compability is default */
+#endif
+#ifndef ZPOS_T
+#define ZPOS_T long /* bw compability is default */
+#endif
+
+#if defined(_INC_WINDOWS) || defined(_WINDOWS_H)
+#define _BOOL_DEFINED
+#endif
+
+#ifndef _BOOL_DEFINED
+#define _BOOL_DEFINED
+typedef signed  int BOOL;
+#endif
+#ifndef FALSE
+#define FALSE 0
+#endif
+#ifndef TRUE
+#define TRUE  1
+#endif
+
+static int fseek_calc(ZPOS_T offset, int origin, ZPOS_T* position, ZPOS_T size)
+{
+    BOOL bOK = TRUE;
+    switch (origin)
+    {
+    case SEEK_SET:
+        //bOK = (offset >= 0) && (offset <= size);
+        if (bOK) *position = offset;
+        break;
+    case SEEK_CUR:
+        bOK = ((offset + *position) >= 0) && (((offset + *position) <= size));
+        if (bOK) *position = offset + *position;
+        break;
+    case SEEK_END:
+        bOK = ((size - offset) >= 0) && (((size - offset) <= size));
+        if (bOK) *position = offset + size - 0;
+        break;
+    default:
+        bOK = FALSE;
+        break;
+    }
+    return bOK ? 0 : -1;
+}
+
+static voidpf ZCALLBACK mem_open OF((
+    voidpf opaque,
+    const char* filename,
+    int mode));
+
+static uLong ZCALLBACK mem_read OF((
+    voidpf opaque,
+    voidpf stream,
+    void* buf,
+    uLong size));
+
+static uLong ZCALLBACK mem_write OF((
+    voidpf opaque,
+    voidpf stream,
+    const void* buf,
+    uLong size));
+
+static ZPOS_T ZCALLBACK mem_tell OF((
+    voidpf opaque,
+    voidpf stream));
+
+static long ZCALLBACK mem_seek OF((
+    voidpf opaque,
+    voidpf stream,
+    ZOFF_T offset,
+    int origin));
+
+static int ZCALLBACK mem_close OF((
+    voidpf opaque,
+    voidpf stream));
+
+static int ZCALLBACK mem_error OF((
+    voidpf opaque,
+    voidpf stream));
+
+typedef struct _MEMFILE
+{
+    void* buffer;    /* Base of the region of memory we're using */
+    ZPOS_T length;   /* Size of the region of memory we're using */
+    ZPOS_T position; /* Current offset in the area */
+} MEMFILE;
+
+static uLong ZCALLBACK mem_read (opaque, stream, buf, size)
+voidpf opaque;
+voidpf stream;
+void* buf;
+uLong size;
+{
+    MEMFILE* handle = (MEMFILE*)stream;
+    /* It's possible for this function to be called with an invalid position.
+     * Additionally, unzip.h minizip uses an unsigned long for the
+     * uncompressed size field, but everwhere else uses a signed long. For
+     * safety, we check here that the handle position is not more than the max
+     * size of a 32-bit signed int.
+     */
+    if (handle->position < 0 || handle->position >= 2147483647)
+    {
+        return 0;
+    }
+
+    if ((handle->position + size) > handle->length)
+    {
+        /* There is a bug in this original code. It's possible for the position
+         * to exceed the size, which results in memcpy being handed a negative
+         * size. See libkml's src/kml/base/zip_file_test.cc for some overflow
+         * tests that exercise this.
+         * size = handle->length - handle->position;
+        */
+        int size_ = handle->length - handle->position;
+        size = (size_ < 0) ? 0 : (uLong)size_;
+    }
+
+    memcpy(buf, ((char*)handle->buffer) + handle->position, size);
+    handle->position += size;
+
+    return size;
+}
+
+static uLong ZCALLBACK mem_write (opaque, stream, buf, size)
+voidpf opaque;
+voidpf stream;
+const void* buf;
+uLong size;
+{
+    MEMFILE* handle = (MEMFILE*)stream;
+
+    if ((handle->position + size) > handle->length)
+    {
+        handle->length = handle->position + size;
+        handle->buffer = realloc(handle->buffer, handle->length);
+    }
+
+    memcpy(((char*)handle->buffer) + handle->position, buf, size);
+    handle->position += size;
+
+    return size;
+}
+
+static ZPOS_T ZCALLBACK mem_tell (opaque, stream)
+voidpf opaque;
+voidpf stream;
+{
+    MEMFILE* handle = (MEMFILE*)stream;
+    return handle->position;
+}
+
+static long ZCALLBACK mem_seek (opaque, stream, offset, origin)
+voidpf opaque;
+voidpf stream;
+ZOFF_T offset;
+int origin;
+{
+    MEMFILE* handle = (MEMFILE*)stream;
+    return fseek_calc(offset, origin, &handle->position, handle->length);
+}
+
+int ZCALLBACK mem_close (opaque, stream)
+voidpf opaque;
+voidpf stream;
+{
+    MEMFILE* handle = (MEMFILE*)stream;
+
+    /* Note that once we've written to the buffer we don't tell anyone
+       about it here. Probably the opaque handle could be used to inform
+       some other component of how much data was written.
+
+       This, and other aspects of writing through this interface, has
+       not been tested.
+     */
+
+    free (handle);
+    return 0;
+}
+
+int ZCALLBACK mem_error (opaque, stream)
+voidpf opaque;
+voidpf stream;
+{
+    /* MEMFILE *handle = (MEMFILE *)stream; */
+    /* We never return errors */
+    return 0;
+}
+
+ZEXTERN void* ZEXPORT mem_simple_create_file(zlib_filefunc_def* api, void* buffer, size_t buf_len)
+{
+    MEMFILE* handle = malloc(sizeof(*handle));
+    api->zopen_file = NULL;
+    api->zread_file = mem_read;
+    api->zwrite_file = mem_write;
+    api->ztell_file = mem_tell;
+    api->zseek_file = mem_seek;
+    api->zclose_file = mem_close;
+    api->zerror_file = mem_error;
+    api->opaque = handle;
+    handle->position = 0;
+    handle->buffer = buffer;
+    handle->length = buf_len;
+    return handle;
+}
+ZEXTERN void ZEXPORT mem_simple_destroy_file(void* handle_ptr)
+{
+    MEMFILE* handle = (MEMFILE*)handle_ptr;
+    free (handle);
+}

examples/ThirdPartyLibs/minizip/ioapi.h

@@ -184,6 +184,8 @@ extern "C"
 #define ZTELL64(filefunc, filestream) (call_ztell64((&(filefunc)), (filestream)))
 #define ZSEEK64(filefunc, filestream, pos, mode) (call_zseek64((&(filefunc)), (filestream), (pos), (mode)))
 
+extern void* ZEXPORT mem_simple_create_file(zlib_filefunc_def* api, void* buffer, size_t buf_len);
+extern void ZEXPORT mem_simple_destroy_file(void* handle_ptr);
 #ifdef __cplusplus
 }
 #endif

examples/ThirdPartyLibs/minizip/unzip.c

@@ -2081,3 +2081,130 @@ extern int ZEXPORT unzSetOffset(unzFile file, uLong pos)
 {
 	return unzSetOffset64(file, pos);
 }
+
+
+ZEXTERN voidpf ZEXPORT unzDetach(file)
+unzFile* file;
+{
+	voidpf stream;
+	unz64_s* s;
+	if (*file == NULL)
+		return NULL;
+	s = (unz64_s*)*file;
+
+	if (s->pfile_in_zip_read != NULL)
+		unzCloseCurrentFile(*file);
+	stream = s->filestream;
+	TRYFREE(s);
+	*file = NULL;
+	return stream;
+}
+
+extern unzFile ZEXPORT unzAttach (stream, pzlib_filefunc_def)
+voidpf stream;
+zlib_filefunc_def* pzlib_filefunc_def;
+{
+	unz64_s us;
+	unz64_s* s;
+	ZPOS64_T central_pos;
+	uLong uL;
+
+	uLong number_disk;          /* number of the current dist, used for
+								   spaning ZIP, unsupported, always 0*/
+	uLong number_disk_with_CD;  /* number the the disk with central dir, used
+								   for spaning ZIP, unsupported, always 0*/
+	uLong number_entry_CD;      /* total number of entries in
+								   the central dir
+								   (same than number_entry on nospan) */
+
+	int err = UNZ_OK;
+
+	if (unz_copyright[0] != ' ')
+		return NULL;
+
+	fill_zlib_filefunc64_32_def_from_filefunc32(&us.z_filefunc,
+		pzlib_filefunc_def);
+
+	us.filestream = stream;
+	if (us.filestream == NULL)
+		return NULL;
+
+	central_pos = unz64local_SearchCentralDir(&us.z_filefunc, us.filestream);
+	if (central_pos == 0)
+		err = UNZ_ERRNO;
+
+	if (ZSEEK64(us.z_filefunc, us.filestream,
+		central_pos, ZLIB_FILEFUNC_SEEK_SET) != 0)
+		err = UNZ_ERRNO;
+
+	/* the signature, already checked */
+	if (unz64local_getLong(&us.z_filefunc, us.filestream, &uL) != UNZ_OK)
+		err = UNZ_ERRNO;
+
+	/* number of this disk */
+	if (unz64local_getShort(&us.z_filefunc, us.filestream, &number_disk) != UNZ_OK)
+		err = UNZ_ERRNO;
+
+	/* number of the disk with the start of the central directory */
+	if (unz64local_getShort(&us.z_filefunc, us.filestream, &number_disk_with_CD) !=
+		UNZ_OK)
+		err = UNZ_ERRNO;
+
+	/* total number of entries in the central dir on this disk */
+	uLong number_entry;
+	if (unz64local_getShort(&us.z_filefunc, us.filestream, &number_entry) !=
+		UNZ_OK)
+		err = UNZ_ERRNO;
+	us.gi.number_entry = number_entry;
+
+	/* total number of entries in the central dir */
+	if (unz64local_getShort(&us.z_filefunc, us.filestream, &number_entry_CD) !=
+		UNZ_OK)
+		err = UNZ_ERRNO;
+
+	if ((number_entry_CD != us.gi.number_entry) ||
+		(number_disk_with_CD != 0) ||
+		(number_disk != 0))
+		err = UNZ_BADZIPFILE;
+
+	/* size of the central directory */
+	uLong size_central_dir;
+	if (unz64local_getLong(&us.z_filefunc, us.filestream, &size_central_dir) !=
+		UNZ_OK)
+		err = UNZ_ERRNO;
+	us.size_central_dir = size_central_dir;
+
+	/* offset of start of central directory with respect to the
+		  starting disk number */
+	uLong offset_central_dir;
+	if (unz64local_getLong(&us.z_filefunc, us.filestream, &offset_central_dir)
+		!= UNZ_OK)
+		err = UNZ_ERRNO;
+	us.offset_central_dir = offset_central_dir;
+
+	/* zipfile comment length */
+	if (unz64local_getShort(&us.z_filefunc, us.filestream, &us.gi.size_comment) !=
+		UNZ_OK)
+		err = UNZ_ERRNO;
+
+	if ((central_pos < us.offset_central_dir + us.size_central_dir) &&
+		(err == UNZ_OK))
+		err = UNZ_BADZIPFILE;
+
+	if (err != UNZ_OK)
+	{
+		return NULL;
+	}
+
+	us.byte_before_the_zipfile = central_pos -
+		(us.offset_central_dir + us.size_central_dir);
+	us.central_pos = central_pos;
+	us.pfile_in_zip_read = NULL;
+	us.encrypted = 0;
+
+
+	s = (unz64_s*)ALLOC(sizeof(unz64_s));
+	*s = us;
+	unzGoToFirstFile((unzFile)s);
+	return (unzFile)s;
+}

examples/ThirdPartyLibs/minizip/unzip.h

@@ -423,6 +423,9 @@ typedef voidp unzFile;
 	extern int ZEXPORT unzSetOffset64(unzFile file, ZPOS64_T pos);
 	extern int ZEXPORT unzSetOffset(unzFile file, uLong pos);
 
+	ZEXTERN voidpf ZEXPORT unzDetach OF((unzFile* file));
+	ZEXTERN unzFile ZEXPORT unzAttach OF((voidpf stream, zlib_filefunc_def*));
+
 #ifdef __cplusplus
 }
 #endif

examples/pybullet/examples/fileIOPlugin.py

@@ -4,6 +4,8 @@ import time
 p.connect(p.GUI)
 fileIO = p.loadPlugin("fileIOPlugin")
 if (fileIO >= 0):
+  #we can have a zipfile (pickup.zip) inside a zipfile (pickup2.zip)
+  p.executePluginCommand(fileIO, "pickup2.zip", [p.AddFileIOAction, p.ZipFileIO])
   p.executePluginCommand(fileIO, "pickup.zip", [p.AddFileIOAction, p.ZipFileIO])
   objs = p.loadSDF("pickup/model.sdf")
   dobot = objs[0]

examples/pybullet/premake4.lua

@@ -16,9 +16,21 @@ project ("pybullet")
 		end
 		
 		includedirs {"../../src", "../../examples",
-		"../../examples/ThirdPartyLibs"}
-		defines {"PHYSICS_IN_PROCESS_EXAMPLE_BROWSER"}
+		"../../examples/ThirdPartyLibs",
+		"../../Extras/VHACD/inc", "../../Extras/VHACD/public",
+		}
+		defines {"BT_ENABLE_VHACD"}
 		
+		defines {"PHYSICS_IN_PROCESS_EXAMPLE_BROWSER"}
+		files 
+		{
+			"../../Extras/VHACD/test/src/main_vhacd.cpp",
+			"../../Extras/VHACD/src/VHACD.cpp",
+			"../../Extras/VHACD/src/vhacdICHull.cpp",
+			"../../Extras/VHACD/src/vhacdManifoldMesh.cpp",
+			"../../Extras/VHACD/src/vhacdMesh.cpp",
+			"../../Extras/VHACD/src/vhacdVolume.cpp",
+		}
 		
 		
 	hasCL = findOpenCL("clew")
@@ -180,8 +192,29 @@ if not _OPTIONS["no-enet"] then
 			"../../examples/SharedMemory/plugins/pdControlPlugin/pdControlPlugin.cpp",
 			"../../examples/SharedMemory/plugins/pdControlPlugin/pdControlPlugin.h",
 		}
-			
-			
+		
+	defines {"B3_ENABLE_FILEIO_PLUGIN", "B3_USE_ZIPFILE_FILEIO"}	
+  files {
+  	"../../examples/SharedMemory/plugins/fileIOPlugin/fileIOPlugin.cpp",
+  	"../../examples/ThirdPartyLibs/minizip/ioapi.c",
+    "../../examples/ThirdPartyLibs/minizip/unzip.c",
+    "../../examples/ThirdPartyLibs/minizip/zip.c",
+    "../../examples/ThirdPartyLibs/zlib/adler32.c",
+    "../../examples/ThirdPartyLibs/zlib/compress.c",
+    "../../examples/ThirdPartyLibs/zlib/crc32.c",
+    "../../examples/ThirdPartyLibs/zlib/deflate.c",
+    "../../examples/ThirdPartyLibs/zlib/gzclose.c",
+    "../../examples/ThirdPartyLibs/zlib/gzlib.c",
+    "../../examples/ThirdPartyLibs/zlib/gzread.c",
+    "../../examples/ThirdPartyLibs/zlib/gzwrite.c",
+    "../../examples/ThirdPartyLibs/zlib/infback.c",
+    "../../examples/ThirdPartyLibs/zlib/inffast.c",
+    "../../examples/ThirdPartyLibs/zlib/inflate.c",
+    "../../examples/ThirdPartyLibs/zlib/inftrees.c",
+    "../../examples/ThirdPartyLibs/zlib/trees.c",
+    "../../examples/ThirdPartyLibs/zlib/uncompr.c",
+    "../../examples/ThirdPartyLibs/zlib/zutil.c",
+  }
 
 	if _OPTIONS["enable_stable_pd"] then
 		defines {"STATIC_LINK_SPD_PLUGIN"}

setup.py

@@ -501,7 +501,7 @@ if 'BT_USE_EGL' in EGL_CXX_FLAGS:
 
 setup(
     name='pybullet',
-    version='2.7.0',
+    version='2.7.1',
     description=
     'Official Python Interface for the Bullet Physics SDK specialized for Robotics Simulation and Reinforcement Learning',
     long_description=