Server: Allow max Tight subrect size to be tweaked

Referring to
https://groups.google.com/g/turbovnc-devel/c/LDSsexdiDXs/m/XC3LxDTpAQAJ
this can be useful with viewers that have a high amount of computational
overhead (e.g. noVNC) running in a loopback environment (e.g. to connect
to a virtual machine.)

This feature is currently undocumented because it benefits only esoteric
use cases, and the full scope of its trade-offs is not well understood.

A better long-term solution is probably pure JPEG encoding (#376), which
noVNC supports, but that would require changes to the TurboJPEG API
(libjpeg-turbo/libjpeg-turbo#777) that haven't landed yet.

unix/Xvnc/programs/Xserver/hw/vnc/rfb.h

@@ -1083,6 +1083,8 @@ static const char subsampStr[TVNC_SAMPOPT][5] = { "1X", "4X", "2X", "Gray" };
 #define TIGHT_DEFAULT_SUBSAMP      TVNC_1X
 #define TIGHT_DEFAULT_QUALITY      95
 
+extern int rfbMaxTightRectSize;
+
 extern int rfbNumCodedRectsTight(rfbClientPtr cl, int x, int y, int w, int h);
 extern Bool rfbSendRectEncodingTight(rfbClientPtr cl, int x, int y, int w,
                                      int h);

unix/Xvnc/programs/Xserver/hw/vnc/rfbserver.c

@@ -487,6 +487,15 @@ static rfbClientPtr rfbNewClient(int sock)
     if (combine > 0 && combine <= 65000) rfbCombineRect = combine;
   }
 
+  if ((env = getenv("TVNC_MAXTIGHTRECTSIZE")) != NULL) {
+    int maxTightRectSize = atoi(env);
+    if (maxTightRectSize > 0) {
+      RFBLOGID("Maximum Tight subrectangle size: %d pixels\n",
+               maxTightRectSize);
+      rfbMaxTightRectSize = maxTightRectSize;
+    }
+  }
+
   cl->firstUpdate = TRUE;
   /* The TigerVNC Viewer won't enable remote desktop resize until it receives
      a desktop resize message from the server, so we give it one with the

unix/Xvnc/programs/Xserver/hw/vnc/tight.c

@@ -91,6 +91,11 @@ static TIGHT_CONF tightConf[4] = {
   { 65536, 2048,  32, 7, 7, 5,  96, 256 }  /* 9 */
 };
 
+int rfbMaxTightRectSize = 0;
+#define MAXRECTSIZE  \
+  (rfbMaxTightRectSize ? rfbMaxTightRectSize :  \
+                         tightConf[compressLevel].maxRectSize)
+
 static int compressLevel;
 static int qualityLevel;
 static int subsampLevel;
@@ -220,7 +225,7 @@ int rfbNumCodedRectsTight(rfbClientPtr cl, int x, int y, int w, int h)
   if (cl->enableLastRectEncoding && w * h >= MIN_SPLIT_RECT_SIZE)
     return 0;
 
-  maxRectSize = tightConf[compressLevel].maxRectSize;
+  maxRectSize = MAXRECTSIZE;
   maxRectWidth = tightConf[compressLevel].maxRectWidth;
 
   if (w > maxRectWidth || w * h > maxRectSize) {
@@ -400,7 +405,7 @@ Bool rfbSendRectEncodingTight(rfbClientPtr cl, int x, int y, int w, int h)
   else
     usePixelFormat24 = FALSE;
 
-  nt = min(rfbNumThreads, w * h / tightConf[compressLevel].maxRectSize);
+  nt = min(rfbNumThreads, w * h / MAXRECTSIZE);
   if (nt < 1) nt = 1;
 
   for (i = 0; i < nt; i++) {
@@ -497,7 +502,7 @@ static Bool SendRectEncodingTight(threadparam *t, int x, int y, int w, int h)
   {
     int maxRectSize, maxRectWidth, nMaxWidth;
 
-    maxRectSize = tightConf[compressLevel].maxRectSize;
+    maxRectSize = MAXRECTSIZE;
     maxRectWidth = tightConf[compressLevel].maxRectWidth;
     nMaxWidth = (w > maxRectWidth) ? maxRectWidth : w;
     nMaxRows = maxRectSize / nMaxWidth;
@@ -741,7 +746,7 @@ static Bool SendRectSimple(threadparam *t, int x, int y, int w, int h)
   int rw, rh;
   rfbClientPtr cl = t->cl;
 
-  maxRectSize = tightConf[compressLevel].maxRectSize;
+  maxRectSize = MAXRECTSIZE;
   maxRectWidth = tightConf[compressLevel].maxRectWidth;
 
   maxBeforeSize = maxRectSize * (cl->format.bitsPerPixel / 8);