vncpasswd add password complexity rule check to enhance security

Use the library pwquality to check password complexity and improve security.
Additionally, optional enable support is also set in CMake.

BUILDING.txt

@@ -55,6 +55,9 @@ Build Requirements (Unix)
    * You might have to enable additional repositories for this. E.g.,
      on RHEL, EPEL and RPMFusion (free + nonfree) need to be enabled.
 
+-- If building vncpasswd with password quality check support:
+   * libpwquality
+
 ============================
 Build Requirements (Windows)
 ============================

unix/vncpasswd/CMakeLists.txt

@@ -1,8 +1,26 @@
+# check for pwquality password check support
+option(ENABLE_PWQUALITY "Enable pwquality password check" ON)
+if(ENABLE_PWQUALITY)
+  if(UNIX)
+    find_package(PkgConfig)
+    if (PKG_CONFIG_FOUND)
+      pkg_check_modules(PWQUALITY pwquality)
+      if(PWQUALITY_FOUND)
+        add_definitions(-DHAVE_PWQUALITY)
+      endif()
+    endif()
+  endif()
+endif()
+
 add_executable(vncpasswd 
   vncpasswd.cxx)
 
 target_include_directories(vncpasswd PUBLIC ${CMAKE_SOURCE_DIR}/common)
 target_link_libraries(vncpasswd tx rfb os)
 
+if(PWQUALITY_FOUND)
+  target_link_libraries(vncpasswd pwquality)
+endif()
+
 install(TARGETS vncpasswd DESTINATION ${CMAKE_INSTALL_FULL_BINDIR})
 install(FILES vncpasswd.man DESTINATION ${CMAKE_INSTALL_FULL_MANDIR}/man1 RENAME vncpasswd.1)

unix/vncpasswd/vncpasswd.cxx

@@ -37,6 +37,9 @@
 
 #include <termios.h>
 
+#ifdef HAVE_PWQUALITY
+#include <pwquality.h>
+#endif
 
 using namespace rfb;
 
@@ -99,6 +102,41 @@ static int encrypt_pipe() {
   return 0;
 }
 
+#ifdef HAVE_PWQUALITY
+static int check_passwd_pwquality(const char *password)
+{
+	int r;
+	void *auxerror;
+	pwquality_settings_t *pwq;
+	pwq = pwquality_default_settings();
+	if (!pwq)
+		return -EINVAL;
+	r = pwquality_read_config(pwq, NULL, &auxerror);
+	if (r) {
+		printf("Cannot check password quality: %s \n",
+			pwquality_strerror(NULL, 0, r, auxerror));
+		pwquality_free_settings(pwq);
+		return -EINVAL;
+	}
+
+	pwquality_set_int_value(pwq, PWQ_SETTING_MIN_LENGTH, 6);
+	pwquality_set_int_value(pwq, PWQ_SETTING_MAX_SEQUENCE, 8);
+	pwquality_set_int_value(pwq, PWQ_SETTING_MAX_REPEAT, 1);
+	pwquality_set_int_value(pwq, PWQ_SETTING_MIN_CLASS, 3);
+
+	r = pwquality_check(pwq, password, NULL, NULL, &auxerror);
+	if (r < 0) {
+		printf("Password quality check failed:\n %s \n",
+			pwquality_strerror(NULL, 0, r, auxerror));
+		r = -EPERM;
+	}
+	pwquality_free_settings(pwq);
+
+	//return the score of password quality
+	return r;
+}
+#endif
+
 static std::vector<uint8_t> readpassword() {
   while (true) {
     const char *passwd = getpassword("Password:");
@@ -116,6 +154,15 @@ static std::vector<uint8_t> readpassword() {
       continue;
     }
 
+#ifdef HAVE_PWQUALITY
+    //the function return score of password quality
+    int r = check_passwd_pwquality(passwd);
+    if (r < 0){
+      printf("Password quality check failed, please set it correctly.\n");
+      continue;
+    }
+#endif
+
     passwd = getpassword("Verify:");
     if (passwd == NULL) {
       perror("getpass error");