embed TCP graphics server in PyBullet for easier working-from-home

This allows to render at a different machine, sending all assets across the TCP network, using port 6667 by default.
You can use ssh port forwarding to get this working:
ssh -R 6667:localhost:6667 username@remotehost.com -v

python3 -m pybullet_utils.graphicsServer
python3 -m pybullet_utils.graphicsClient

Note that there are still some tcp networking issues that can cause a hang, just restart graphics server and graphics client if it hangs.
This commit is contained in:
Erwin Coumans
2020-03-20 15:43:52 -07:00
parent c06859723b
commit 344bd629ee
15 changed files with 1421 additions and 25 deletions

View File

@@ -11,10 +11,557 @@
#include "GraphicsSharedMemoryBlock.h"
#include "../CommonInterfaces/CommonGUIHelperInterface.h"
#include "SharedMemoryPublic.h"
#include "../MultiThreading/b3ThreadSupportInterface.h"
#include "Utils/b3Clock.h"
#ifdef BT_ENABLE_CLSOCKET
#include "PassiveSocket.h" // Include header for active socket object definition
#include <stdio.h>
#include "../CommonInterfaces/CommonGUIHelperInterface.h"
#include "Bullet3Common/b3CommandLineArgs.h"
#include "RemoteGUIHelper.h"
#include "GraphicsSharedMemoryPublic.h"
#include "GraphicsSharedMemoryCommands.h"
bool gVerboseNetworkMessagesServer = false;
void MySerializeInt(unsigned int sz, unsigned char* output)
{
unsigned int tmp = sz;
output[0] = tmp & 255;
tmp = tmp >> 8;
output[1] = tmp & 255;
tmp = tmp >> 8;
output[2] = tmp & 255;
tmp = tmp >> 8;
output[3] = tmp & 255;
}
void submitStatus(CActiveSocket* pClient, GraphicsSharedMemoryStatus& serverStatus, b3AlignedObjectArray<char>& buffer)
{
b3AlignedObjectArray<unsigned char> packetData;
unsigned char* statBytes = (unsigned char*)&serverStatus;
//create packetData with [int packetSizeInBytes, status, streamBytes)
packetData.resize(4 + sizeof(GraphicsSharedMemoryStatus) + serverStatus.m_numDataStreamBytes);
int sz = packetData.size();
int curPos = 0;
if (gVerboseNetworkMessagesServer)
{
//printf("buffer.size = %d\n", buffer.size());
printf("serverStatus packed size = %d\n", sz);
}
MySerializeInt(sz, &packetData[curPos]);
curPos += 4;
for (int i = 0; i < sizeof(GraphicsSharedMemoryStatus); i++)
{
packetData[i + curPos] = statBytes[i];
}
curPos += sizeof(GraphicsSharedMemoryStatus);
for (int i = 0; i < serverStatus.m_numDataStreamBytes; i++)
{
packetData[i + curPos] = buffer[i];
}
pClient->Send(&packetData[0], packetData.size());
if (gVerboseNetworkMessagesServer)
printf("pClient->Send serverStatus: %d\n", packetData.size());
}
#endif //BT_ENABLE_CLSOCKET
#define MAX_GRAPHICS_SHARED_MEMORY_BLOCKS 1
struct TCPArgs
{
TCPArgs()
: m_cs(0),
m_port(6667)
{
}
b3CriticalSection* m_cs;
int m_port;
};
struct TCPThreadLocalStorage
{
int threadId;
};
enum TCPCommunicationEnums
{
eTCPRequestTerminate = 11,
eTCPIsUnInitialized,
eTCPIsInitialized,
eTCPInitializationFailed,
eTCPHasTerminated
};
void TCPThreadFunc(void* userPtr, void* lsMemory)
{
printf("TCPThreadFunc thread started\n");
TCPArgs* args = (TCPArgs*)userPtr;
//int workLeft = true;
b3Clock clock;
clock.reset();
b3Clock sleepClock;
bool init = true;
if (init)
{
unsigned int cachedSharedParam = eTCPIsInitialized;
args->m_cs->lock();
args->m_cs->setSharedParam(0, eTCPIsInitialized);
args->m_cs->unlock();
double deltaTimeInSeconds = 0;
int numCmdSinceSleep1ms = 0;
unsigned long long int prevTime = clock.getTimeMicroseconds();
#ifdef BT_ENABLE_CLSOCKET
b3Clock clock;
double timeOutInSeconds = 10;
RemoteGUIHelper guiHelper;
bool isPhysicsClientConnected = guiHelper.isConnected();
bool exitRequested = false;
b3AlignedObjectArray< b3AlignedObjectArray<char> > slots;
int maxSlots = 10;
slots.resize(maxSlots);
if (!isPhysicsClientConnected)
{
printf("TCP thread error connecting to shared memory. Machine needs a reboot?\n");
}
btAssert(isPhysicsClientConnected);
printf("Starting TCP server using port %d\n", args->m_port);
CPassiveSocket socket;
CActiveSocket* pClient = NULL;
//--------------------------------------------------------------------------
// Initialize our socket object
//--------------------------------------------------------------------------
socket.Initialize();
socket.Listen("localhost", args->m_port);
//socket.SetReceiveTimeout(1, 0);
//socket.SetNonblocking();
int curNumErr = 0;
#endif
do
{
{
b3Clock::usleep(0);
}
///////////////////////////////
#ifdef BT_ENABLE_CLSOCKET
{
b3Clock::usleep(0);
if ((pClient = socket.Accept()) != NULL)
{
b3AlignedObjectArray<char> bytesReceived;
int clientPort = socket.GetClientPort();
if (gVerboseNetworkMessagesServer)
printf("connected from %s:%d\n", socket.GetClientAddr(), clientPort);
if (pClient->Receive(4))
{
int clientKey = *(int*)pClient->GetData();
if (clientKey == GRAPHICS_SHARED_MEMORY_MAGIC_NUMBER)
{
printf("Client version OK %d\n", clientKey);
}
else
{
printf("Server version (%d) mismatches Client Version (%d)\n", GRAPHICS_SHARED_MEMORY_MAGIC_NUMBER, clientKey);
continue;
}
}
//----------------------------------------------------------------------
// Receive request from the client.
//----------------------------------------------------------------------
while (cachedSharedParam != eTCPRequestTerminate)
{
//printf("try receive\n");
bool receivedData = false;
int maxLen = 4 + sizeof(GraphicsSharedMemoryCommand) + GRAPHICS_SHARED_MEMORY_MAX_STREAM_CHUNK_SIZE;
if (pClient->Receive(maxLen))
{
//heuristic to detect disconnected clients
CSimpleSocket::CSocketError err = pClient->GetSocketError();
if (err != CSimpleSocket::SocketSuccess || !pClient->IsSocketValid())
{
b3Clock::usleep(100);
curNumErr++;
if (curNumErr > 100)
{
printf("TCP Connection error = %d, curNumErr = %d\n", (int)err, curNumErr);
break;
}
}
curNumErr = 0;
char* msg2 = (char*)pClient->GetData();
int numBytesRec2 = pClient->GetBytesReceived();
if (numBytesRec2 < 0)
{
numBytesRec2 = 0;
}
int curSize = bytesReceived.size();
bytesReceived.resize(bytesReceived.size() + numBytesRec2);
for (int i = 0; i < numBytesRec2; i++)
{
bytesReceived[curSize + i] = msg2[i];
}
if (bytesReceived.size() >= 4)
{
int numBytesRec = bytesReceived.size();
if (numBytesRec >= 10)
{
if (strncmp(&bytesReceived[0], "disconnect", 10) == 0)
{
printf("Disconnect request received\n");
bytesReceived.clear();
break;
}
}
if (gVerboseNetworkMessagesServer)
{
printf("received message length [%d]\n", numBytesRec);
}
receivedData = true;
GraphicsSharedMemoryCommand cmd;
GraphicsSharedMemoryCommand* cmdPtr = 0;
int type = *(int*)&bytesReceived[0];
//performance test
if (numBytesRec == sizeof(int))
{
cmdPtr = &cmd;
cmd.m_type = *(int*)&bytesReceived[0];
}
else
{
if (numBytesRec == sizeof(GraphicsSharedMemoryCommand))
{
cmdPtr = (GraphicsSharedMemoryCommand*)&bytesReceived[0];
}
else
{
if (numBytesRec == 36)
{
cmdPtr = &cmd;
memcpy(&cmd, &bytesReceived[0], numBytesRec);
}
}
}
if (cmdPtr)
{
GraphicsSharedMemoryStatus serverStatus;
serverStatus.m_type = GFX_CMD_CLIENT_COMMAND_FAILED;
serverStatus.m_numDataStreamBytes = 0;
b3AlignedObjectArray<char> buffer;
buffer.resize(GRAPHICS_SHARED_MEMORY_MAX_STREAM_CHUNK_SIZE);
bool hasStatus = true;
if (gVerboseNetworkMessagesServer)
printf("processing command:");
switch (cmdPtr->m_type)
{
case GFX_CMD_0:
{
int axis = cmdPtr->m_upAxisYCommand.m_enableUpAxisY ? 1 : 2;
guiHelper.setUpAxis(axis);
serverStatus.m_type = GFX_CMD_CLIENT_COMMAND_COMPLETED;
if (gVerboseNetworkMessagesServer)
printf("GFX_CMD_0\n");
break;
}
case GFX_CMD_SET_VISUALIZER_FLAG:
{
guiHelper.setVisualizerFlag(
cmdPtr->m_visualizerFlagCommand.m_visualizerFlag,
cmdPtr->m_visualizerFlagCommand.m_enable);
serverStatus.m_type = GFX_CMD_CLIENT_COMMAND_COMPLETED;
if (gVerboseNetworkMessagesServer)
printf("GFX_CMD_SET_VISUALIZER_FLAG\n");
break;
}
case GFX_CMD_UPLOAD_DATA:
{
int slot = cmdPtr->m_uploadDataCommand.m_dataSlot;
submitStatus(pClient, serverStatus, buffer);
if (gVerboseNetworkMessagesServer)
printf("GFX_CMD_UPLOAD_DATA receiving data\n");
if (pClient->Receive(cmdPtr->m_uploadDataCommand.m_numBytes))
{
//heuristic to detect disconnected clients
CSimpleSocket::CSocketError err = pClient->GetSocketError();
if (err != CSimpleSocket::SocketSuccess || !pClient->IsSocketValid())
{
curNumErr++;
printf("TCP Connection error = %d, curNumErr = %d\n", (int)err, curNumErr);
}
char* msg2 = (char*)pClient->GetData();
int numBytesRec2 = pClient->GetBytesReceived();
if (gVerboseNetworkMessagesServer)
printf("received %d bytes\n", numBytesRec2);
slots[slot].resize(numBytesRec2);
for (int i = 0; i < numBytesRec2; i++)
{
slots[slot][i] = msg2[i];
}
}
serverStatus.m_type = GFX_CMD_CLIENT_COMMAND_COMPLETED;
if (gVerboseNetworkMessagesServer)
printf("GFX_CMD_UPLOAD_DATA\n");
break;
}
case GFX_CMD_REGISTER_TEXTURE:
{
const unsigned char* texels = (const unsigned char*)&slots[0][0];
serverStatus.m_registerTextureStatus.m_textureId = guiHelper.registerTexture(texels, cmdPtr->m_registerTextureCommand.m_width,
cmdPtr->m_registerTextureCommand.m_height);
serverStatus.m_type = GFX_CMD_REGISTER_TEXTURE_COMPLETED;
if (gVerboseNetworkMessagesServer)
printf("GFX_CMD_REGISTER_TEXTURE\n");
break;
}
case GFX_CMD_REGISTER_GRAPHICS_SHAPE:
{
const float* vertices = (const float*)&slots[0][0];
const int* indices = (const int*)&slots[1][0];
serverStatus.m_registerGraphicsShapeStatus.m_shapeId = guiHelper.registerGraphicsShape(vertices, cmdPtr->m_registerGraphicsShapeCommand.m_numVertices, indices,
cmdPtr->m_registerGraphicsShapeCommand.m_numIndices, cmdPtr->m_registerGraphicsShapeCommand.m_primitiveType,
cmdPtr->m_registerGraphicsShapeCommand.m_textureId);
serverStatus.m_type = GFX_CMD_REGISTER_GRAPHICS_SHAPE_COMPLETED;
if (gVerboseNetworkMessagesServer)
printf("GFX_CMD_REGISTER_GRAPHICS_SHAPE\n");
break;
}
case GFX_CMD_REGISTER_GRAPHICS_INSTANCE:
{
serverStatus.m_registerGraphicsInstanceStatus.m_graphicsInstanceId =
guiHelper.registerGraphicsInstance(
cmdPtr->m_registerGraphicsInstanceCommand.m_shapeIndex,
cmdPtr->m_registerGraphicsInstanceCommand.m_position,
cmdPtr->m_registerGraphicsInstanceCommand.m_quaternion,
cmdPtr->m_registerGraphicsInstanceCommand.m_color,
cmdPtr->m_registerGraphicsInstanceCommand.m_scaling);
serverStatus.m_type = GFX_CMD_REGISTER_GRAPHICS_INSTANCE_COMPLETED;
if (gVerboseNetworkMessagesServer)
printf("GFX_CMD_REGISTER_GRAPHICS_INSTANCE\n");
break;
}
case GFX_CMD_SYNCHRONIZE_TRANSFORMS:
{
const GUISyncPosition* positions = (const GUISyncPosition*)&slots[0][0];
guiHelper.syncPhysicsToGraphics2(positions, cmdPtr->m_syncTransformsCommand.m_numPositions);
serverStatus.m_type = GFX_CMD_CLIENT_COMMAND_COMPLETED;
if (gVerboseNetworkMessagesServer)
printf("GFX_CMD_SYNCHRONIZE_TRANSFORMS\n");
break;
}
case GFX_CMD_REMOVE_ALL_GRAPHICS_INSTANCES:
{
guiHelper.removeAllGraphicsInstances();
serverStatus.m_type = GFX_CMD_CLIENT_COMMAND_COMPLETED;
if (gVerboseNetworkMessagesServer)
printf("GFX_CMD_REMOVE_ALL_GRAPHICS_INSTANCES\n");
break;
}
case GFX_CMD_REMOVE_SINGLE_GRAPHICS_INSTANCE:
{
guiHelper.removeGraphicsInstance(cmdPtr->m_removeGraphicsInstanceCommand.m_graphicsUid);
serverStatus.m_type = GFX_CMD_CLIENT_COMMAND_COMPLETED;
if (gVerboseNetworkMessagesServer)
printf("GFX_CMD_REMOVE_SINGLE_GRAPHICS_INSTANCE\n");
break;
}
case GFX_CMD_CHANGE_RGBA_COLOR:
{
guiHelper.changeRGBAColor(cmdPtr->m_changeRGBAColorCommand.m_graphicsUid,
cmdPtr->m_changeRGBAColorCommand.m_rgbaColor);
serverStatus.m_type = GFX_CMD_CLIENT_COMMAND_COMPLETED;
if (gVerboseNetworkMessagesServer)
printf("GFX_CMD_CHANGE_RGBA_COLOR\n");
break;
}
case GFX_CMD_GET_CAMERA_INFO:
{
//bool RemoteGUIHelper::getCameraInfo(int* width, int* height,
// float viewMatrix[16], float projectionMatrix[16],
// float camUp[3], float camForward[3], float hor[3], float vert[3],
// float* yaw, float* pitch, float* camDist, float camTarget[3]) const
guiHelper.getCameraInfo(&serverStatus.m_getCameraInfoStatus.width,
&serverStatus.m_getCameraInfoStatus.height,
serverStatus.m_getCameraInfoStatus.viewMatrix,
serverStatus.m_getCameraInfoStatus.projectionMatrix,
serverStatus.m_getCameraInfoStatus.camUp,
serverStatus.m_getCameraInfoStatus.camForward,
serverStatus.m_getCameraInfoStatus.hor,
serverStatus.m_getCameraInfoStatus.vert,
&serverStatus.m_getCameraInfoStatus.yaw,
&serverStatus.m_getCameraInfoStatus.pitch,
&serverStatus.m_getCameraInfoStatus.camDist,
serverStatus.m_getCameraInfoStatus.camTarget);
serverStatus.m_type = GFX_CMD_GET_CAMERA_INFO_COMPLETED;
if (gVerboseNetworkMessagesServer)
printf("GFX_CMD_GET_CAMERA_INFO\n");
break;
}
case GFX_CMD_INVALID:
case GFX_CMD_MAX_CLIENT_COMMANDS:
default:
{
printf("UNKNOWN COMMAND!\n");
btAssert(0);
hasStatus = false;
}
}
double startTimeSeconds = clock.getTimeInSeconds();
double curTimeSeconds = clock.getTimeInSeconds();
if (gVerboseNetworkMessagesServer)
{
//printf("buffer.size = %d\n", buffer.size());
printf("serverStatus.m_numDataStreamBytes = %d\n", serverStatus.m_numDataStreamBytes);
}
if (hasStatus)
{
submitStatus(pClient, serverStatus, buffer);
}
bytesReceived.clear();
}
else
{
//likely an incomplete packet, let's append more bytes
//printf("received packet with unknown contents\n");
}
}
}
if (!receivedData)
{
//printf("Didn't receive data.\n");
}
}
printf("Disconnecting client.\n");
pClient->Close();
delete pClient;
}
}
#endif //BT_ENABLE_CLSOCKET
///////////////////////////////
args->m_cs->lock();
cachedSharedParam = args->m_cs->getSharedParam(0);
args->m_cs->unlock();
} while (cachedSharedParam != eTCPRequestTerminate);
socket.Close();
socket.Shutdown(CSimpleSocket::Both);
}
else
{
args->m_cs->lock();
args->m_cs->setSharedParam(0, eTCPInitializationFailed);
args->m_cs->unlock();
}
printf("TCPThreadFunc thread exit\n");
//do nothing
}
void* TCPlsMemoryFunc()
{
//don't create local store memory, just return 0
return new TCPThreadLocalStorage;
}
void TCPlsMemoryReleaseFunc(void* ptr)
{
TCPThreadLocalStorage* p = (TCPThreadLocalStorage*)ptr;
delete p;
}
#ifndef _WIN32
#include "../MultiThreading/b3PosixThreadSupport.h"
b3ThreadSupportInterface* createTCPThreadSupport(int numThreads)
{
b3PosixThreadSupport::ThreadConstructionInfo constructionInfo("TCPThreads",
TCPThreadFunc,
TCPlsMemoryFunc,
TCPlsMemoryReleaseFunc,
numThreads);
b3ThreadSupportInterface* threadSupport = new b3PosixThreadSupport(constructionInfo);
return threadSupport;
}
#elif defined(_WIN32)
#include "../MultiThreading/b3Win32ThreadSupport.h"
b3ThreadSupportInterface* createTCPThreadSupport(int numThreads)
{
b3Win32ThreadSupport::Win32ThreadConstructionInfo threadConstructionInfo("TCPThreads", TCPThreadFunc, TCPlsMemoryFunc, TCPlsMemoryReleaseFunc, numThreads);
b3Win32ThreadSupport* threadSupport = new b3Win32ThreadSupport(threadConstructionInfo);
return threadSupport;
}
#endif
class GraphicsServerExample : public CommonExampleInterface
{
CommonGraphicsApp* m_app;
@@ -29,7 +576,10 @@ class GraphicsServerExample : public CommonExampleInterface
float m_x;
float m_y;
float m_z;
b3ThreadSupportInterface* m_threadSupport;
TCPArgs m_args;
public:
GraphicsServerExample(GUIHelperInterface* guiHelper)
: m_guiHelper(guiHelper),
@@ -42,6 +592,7 @@ public:
m_app = guiHelper->getAppInterface();
m_app->setUpAxis(2);
for (int i = 0; i < MAX_GRAPHICS_SHARED_MEMORY_BLOCKS; i++)
{
@@ -54,23 +605,54 @@ public:
m_sharedMemory = new PosixSharedMemory();
#endif
#if 0
{
int boxId = m_app->registerCubeShape(0.1, 0.1, 0.1);
btVector3 pos(0, 0, 0);
btQuaternion orn(0, 0, 0, 1);
btVector4 color(0.3, 0.3, 0.3, 1);
btVector3 scaling(1, 1, 1);
m_app->m_renderer->registerGraphicsInstance(boxId, pos, orn, color, scaling);
}
m_app->m_renderer->writeTransforms();
#endif
connectSharedMemory(m_guiHelper, m_sharedMemoryKey);
m_threadSupport = createTCPThreadSupport(1);
m_args.m_cs = m_threadSupport->createCriticalSection();
m_args.m_cs->setSharedParam(0, eTCPIsUnInitialized);
m_threadSupport->runTask(B3_THREAD_SCHEDULE_TASK, (void*)&this->m_args, 0);
bool isUninitialized = true;
while (isUninitialized)
{
m_args.m_cs->lock();
isUninitialized = (m_args.m_cs->getSharedParam(0) == eTCPIsUnInitialized);
m_args.m_cs->unlock();
#ifdef _WIN32
b3Clock::usleep(1000);
#endif
}
}
virtual ~GraphicsServerExample()
{
m_args.m_cs->setSharedParam(0, eTCPRequestTerminate);
int numActiveThreads = 1;
while (numActiveThreads)
{
int arg0, arg1;
if (m_threadSupport->isTaskCompleted(&arg0, &arg1, 0))
{
numActiveThreads--;
printf("numActiveThreads = %d\n", numActiveThreads);
}
else
{
b3Clock::usleep(0);
}
};
m_threadSupport->deleteCriticalSection(m_args.m_cs);
delete m_threadSupport;
m_threadSupport = 0;
disconnectSharedMemory();
}