Updating Application That Has Custom Actions in Setup.dll

Recently I was creating installer for application that is suppose to be updated regulary. Also cab installer for this application used setup.dll to perform custom actions during install/uninstall.

Since cab installer doesn't have "update" feature installing update is done as a sequence: uninstall_old > install_new. Calls to setup.dll will be as follows:

1. Install_Init() from new setup.dll is called with fFirstCall=true and fPreviouslyInstalled=true.
2. Old installation is removed. This leads to calls to Uninstall_Init() and Uninstall_Exist() from old setup.dll (which is stored in \windows\AppMgr).
3. Install_Init() from new setup dll is called with fFirstCall=false and fPreviouslyInstalled=false.
4. New files are extracted by cab installer.
5. Install_Exist() is called.

To control updating process I needed to use some kind of a marker that will be available at all of those steps.

Originally I tried to use named Mutex (mainly because it is an in-memory object), but for some reason it didn't work as expected. So I considered key in registry and a file. Creating/removing file just to use it as a marker is not very good idea and my final solution was to use a key in registry:

#include <windows.h> 
#include <string.h>

///////////////////////////////////////////////////////////
// Methods to work with marker.

void SetInstallingStartFlag() {
 HKEY hKey = NULL;
 DWORD dwDisposition;
 if (::RegCreateKeyEx(HKEY_CURRENT_USER, _T("\\SOFTWARE\\MySoftwareInstall"), 0, NULL, REG_OPTION_NON_VOLATILE, KEY_ALL_ACCESS, NULL, &hKey, &dwDisposition) == ERROR_SUCCESS) {
  ::RegCloseKey(hKey);
 }
}
void RemoveInstallingStartFlag() {
 HKEY hKey = NULL;
 if (::RegOpenKeyEx(HKEY_CURRENT_USER, _T(""), 0, KEY_ALL_ACCESS, &hKey) == ERROR_SUCCESS) {
  ::RegDeleteKey(hKey, _T("\\SOFTWARE\\MySoftwareInstall"));
  ::RegCloseKey(hKey);
 }
}
BOOL OpenFlagKey() {
 BOOL bOpened = FALSE;

 HKEY hKey = NULL;
 if (::RegOpenKeyEx(HKEY_CURRENT_USER, _T("\\SOFTWARE\\MySoftwareInstall"), 0, KEY_READ, &hKey) == ERROR_SUCCESS) {
  bOpened = TRUE;
  ::RegCloseKey(hKey);
 }

 return bOpened;
}
BOOL IsRemovingDuringUpdate() {
 return OpenFlagKey();
}
BOOL IsInstallingDuringUpdate() {
 return (OpenFlagKey() == FALSE);
}


///////////////////////////////////////////////////////////
// Setup.dll interface methods and enums.

typedef enum {
 codeINSTALL_INIT_CONTINUE = 0,
 codeINSTALL_INIT_CANCEL
}
codeINSTALL_INIT;

typedef enum {
 codeINSTALL_EXIT_DONE = 0,
 codeINSTALL_EXIT_UNINSTALL
}
codeINSTALL_EXIT;

typedef enum {
 codeUNINSTALL_INIT_CONTINUE = 0,
 codeUNINSTALL_INIT_CANCEL
}
codeUNINSTALL_INIT;
typedef enum {
 codeUNINSTALL_EXIT_DONE = 0
}
codeUNINSTALL_EXIT;

extern "C" __declspec(dllexport) codeINSTALL_INIT Install_Init(HWND hwndParent, BOOL fFirstCall, BOOL fPreviouslyInstalled, LPCTSTR pszInstallDir) {
 if (fFirstCall == TRUE) {
  SetInstallingStartFlag();
 }

 return codeINSTALL_INIT_CONTINUE;
}

extern "C" __declspec(dllexport) codeINSTALL_EXIT Install_Exit(HWND hwndParent, LPCTSTR pszInstallDir, WORD cFailedDirs, WORD cFailedFiles, WORD cFailedRegKeys, WORD cFailedRegVals, WORD cFailedShortcuts) {
 BOOL bUpdating = IsInstallingDuringUpdate();
 RemoveInstallingStartFlag();
 
 if (bUpdating == TRUE) {
  // Run updating code here...
 }
 else {
  // Run installation code here...
 }

 return codeINSTALL_INIT_CONTINUE;
}

extern "C" __declspec(dllexport) codeUNINSTALL_INIT Uninstall_Init(HWND hwndParent, LPCTSTR pszInstallDir) {
 BOOL bUpdating = IsRemovingDuringUpdate();
 RemoveInstallingStartFlag();
 if (bUpdating == FALSE) {
  // Run uninstallation code here...
 }
 
 return codeUNINSTALL_INIT_CONTINUE;
}

extern "C" __declspec(dllexport) codeUNINSTALL_EXIT Uninstall_Exit(HWND hwndParent) {
 return codeUNINSTALL_EXIT_DONE;
}