FileOpen

/* fileSaveAs.c

This program reads a pdf encrypted with the fileOpen plugin,
and attempts to write a plain pdf.

Compile with:
gcc -g -o fileSaveAs.exe -lssl -lcrypto fileSaveAs.c

To compile, you'll need packages libssl09-dev and libCrypto.
This is a list of things that must be worked out:

1) It does not decrypt strings, so there is no navigation bar on the left, no author, no additional info and such.
2) Often in the resulting pdf there are many white pages.
3) I had only one ebook to work with, so the salt that I have blatantly hardcoded may be different.
4) The ebook I have contains the strings '/FOPN_fLock /V 1', I guess that it is the version of the plugin. There certainly are other versions of the plugin, I don't know about them.
5) When printing, sometimes Acrobat Reader says that there are illegal characters.

So much to do...
*/

#include <stdio.h>
#include <openssl/md5.h>
#include <sys/stat.h>
#include <sys/fcntl.h>
#include <getopt.h>

#define MAX_OBJ_LEN 2000000
#define STREAM_STR "stream"
#define END_STREAM_STR "endstream"
#define OBJ_STR " obj"
#define END_OBJ_DELIM "endobj"
#define ENCRYPTION_STRING "/Encrypt"

/* I know this is ugly, but hey, I said there is much to do! Begin here! */
unsigned char pObjBuf[MAX_OBJ_LEN];

#define SHUFFLEUP(YY, XX, ZZ, AA) { \
YY=0; \
YY=(char)*ZZ; \
YY=YY << XX; \
AA |= YY; \
ZZ++; }

unsigned char S[256];
unsigned char K[256];

/* This is the only thing they added on top of plain RC4. */
void superSecret (unsigned char *pData, int nIntParam, MD5_CTX *pMd5Ctx)
{
unsigned long lTmp1,
lTmp2,
lTmp3,
lTmp4,
lTmp5,
lTmp6,
lTmp7;
unsigned long *plPoint1;
unsigned char *plMainPnt;

plMainPnt=pData;
if(0 == nIntParam)
return;
lTmp4=pMd5Ctx->Nl;
lTmp7=nIntParam;
lTmp7=lTmp7<<3;
lTmp4+= lTmp7;
lTmp4 &= 0xFF;
if(lTmp4<pMd5Ctx->Nl)
pMd5Ctx->Nh++;
lTmp6=nIntParam;
lTmp6=lTmp6 >> 0x1d;
pMd5Ctx->Nh+= lTmp6;
pMd5Ctx->Nl=lTmp4;

if(0!= pMd5Ctx->num) {
plPoint1=(unsigned long *)pMd5Ctx->data;
lTmp5=pMd5Ctx->num;
lTmp5=lTmp5 >> 2;
lTmp7=pMd5Ctx->num;
lTmp7&= 3;
lTmp1=lTmp7;
lTmp4=pMd5Ctx->num;
lTmp4+= nIntParam;
if(lTmp4>= 0x40) {
lTmp4=plPoint1[lTmp5];
switch(lTmp1) {
case 0:
SHUFFLEUP(lTmp4, 0, plMainPnt, lTmp4);
case 1:
SHUFFLEUP(lTmp7, 8, plMainPnt, lTmp4);
case 2:
SHUFFLEUP(lTmp6,0x10, plMainPnt, lTmp4);
case 3:
SHUFFLEUP(lTmp7, 0x18, plMainPnt, lTmp4);
}
plPoint1[lTmp5]=lTmp4;

lTmp5++;
if(lTmp5 <0x10) {
do {
SHUFFLEUP(lTmp4, 0, plMainPnt, lTmp4);
SHUFFLEUP(lTmp6, 8, plMainPnt, lTmp4);
SHUFFLEUP(lTmp7, 0x10, plMainPnt, lTmp4);
SHUFFLEUP(lTmp6, 0x18, plMainPnt, lTmp4);
plPoint1[lTmp5]=lTmp4;
} while(lTmp5<10);
}
lTmp7=0x40-pMd5Ctx->num;
nIntParam-= lTmp7;
MD5_Update (pMd5Ctx, (unsigned char *)plPoint1, 0x40);
pMd5Ctx->num;
} else {
pMd5Ctx->num+= nIntParam;
if(lTmp1+nIntParam<4) {
lTmp4=plPoint1[lTmp5];
switch(lTmp1) {
case 0:
SHUFFLEUP(lTmp4, 0, plMainPnt, lTmp4);
nIntParam--;
if(0 == nIntParam)
break;
case 1:
SHUFFLEUP(lTmp7, 8, plMainPnt, lTmp4);
nIntParam--;
if(0 == nIntParam)
break;
case 2:
SHUFFLEUP(lTmp6, 0x10, plMainPnt, lTmp4);
}
plPoint1[lTmp5]=lTmp4;
return;
}
lTmp7=pMd5Ctx->num;
lTmp7=lTmp7 >> 2;
lTmp3=lTmp7;
lTmp2=pMd5Ctx->num & 3;
lTmp4=((unsigned char *)plPoint1)[4*lTmp5];

switch(lTmp1) {
case 0:
SHUFFLEUP(lTmp4, 0, plMainPnt, lTmp4);
case 1:
SHUFFLEUP(lTmp7, 8, plMainPnt, lTmp4);
case 2:
SHUFFLEUP(lTmp6, 0x10, plMainPnt, lTmp4);
case 3:
SHUFFLEUP(lTmp7, 0x18, plMainPnt, lTmp4);
}
plPoint1[lTmp5++]=lTmp4;
if(lTmp5<lTmp3) {
do {
SHUFFLEUP(lTmp4, 0, plMainPnt, lTmp4);
SHUFFLEUP(lTmp6, 8, plMainPnt, lTmp4);
SHUFFLEUP(lTmp7, 0x10, plMainPnt, lTmp4);
SHUFFLEUP(lTmp6, 0x18, plMainPnt, lTmp4);
plPoint1[lTmp5++]=lTmp4;
} while(lTmp5<lTmp3);
}
if(0 == lTmp2)
return;
lTmp4=0;
plMainPnt+= lTmp2;
switch(lTmp2) {
case 0:
plMainPnt--;
lTmp4=0;
lTmp4=(char)*plMainPnt;
lTmp4 << 0x10;
case 2:
plMainPnt--;
lTmp6=0;
lTmp6=(char)*plMainPnt;
lTmp6<<8;
lTmp4|= lTmp6;
case 1:
plMainPnt--;
lTmp7=0;
lTmp7=(char)*plMainPnt;
lTmp4|= lTmp7;
case 3:
plPoint1[lTmp5]=lTmp4;
return;
}
}
}
if(0 == ((unsigned long)plMainPnt & 0x3)) {
lTmp5=nIntParam;
if ((lTmp5>> 6) >0) {
lTmp5=lTmp5<<6;
MD5_Update (pMd5Ctx, (unsigned char *)plMainPnt, lTmp5);
plMainPnt+= lTmp5;
nIntParam-= lTmp5;
}
}
plPoint1=(unsigned long *)pMd5Ctx->data;
if(nIntParam>= 0x40) {
do {
if(plMainPnt!= (unsigned char *)plPoint1) {
memcpy((unsigned char *)plPoint1, plMainPnt, 0x40);
}
plMainPnt+= 0x40;
MD5_Update (pMd5Ctx, (unsigned char *)pMd5Ctx->data, 0x40);
} while(nIntParam>= 0x40);
}
lTmp1=nIntParam;
pMd5Ctx->num=lTmp1;
if(0 == lTmp1)
return;
lTmp5=lTmp1>>2;
plPoint1[lTmp5]=0;
memcpy((unsigned char *)plPoint1, plMainPnt, lTmp1);
}

/* From 'Applied Cryptography'.
Yes, they used an algorithm from a book. I can understand: after all, it is a good algorithm.
But If I wanted to distribute some encrypted data, I wouldn't put the algorithm _AND_ the key in the same place. */

void CreateRc4Sbox
(
unsigned char *pKeySeed
)
{
int i=0;
int j=0;
int nRealIndex;
unsigned char nTemp;

for (i=0; i<256; i++)
S[i]=i;

for (i=0; i<256; i++)
K[i]=pKeySeed[i%16];
nRealIndex=0;
for (i=0; i<256; i++) {
j=(j + S[i] + K[nRealIndex])%256;
nRealIndex++;
if(nRealIndex>9)
nRealIndex=0;
nTemp=S[i];
S[i]=S[j];
S[j]=nTemp;
}
}

/* Also from the book */
int Rc4Decrypt
(
char *pCryptoData,
int nCryptoBufLen,
char *pResults,
int nResultsBufSize,
unsigned char *SBox
)
{
short nByte;
int nClearPos=0,
nBuf=0,
nBuf2=0,
nCount,
nSwap,
nLength=nCryptoBufLen,
nAdv;
while(0!= nLength) {
nBuf++;
nBuf &= 0xFF;
nCount=SBox[nBuf];
nBuf2 += nCount;
nBuf2 &= 0xFF;
nSwap=SBox[nBuf2];
nAdv=nSwap;
SBox[nBuf]=nSwap;
SBox[nBuf2]=nCount;
nCount += nAdv;
nCount &= 0xFF;
nByte=SBox[nCount];
nByte ^= *pCryptoData;
nClearPos++;
*pResults=nByte;
++pCryptoData;
++pResults;
nLength--;
}
return nClearPos;
}

/* this function calculates the key and executes the RC4 decrypt loop. Here I have hardcoded the salt: 'DgNab'.
It may change in other ebooks but I only have one... */

int mainDecryptLoop (unsigned char * pCryptText, int nCryptSize, unsigned char * pClearBuffer, int nClearBufferSize, int nNumObj)
{
unsigned char rc4Key[16],
keyBuf[8];
int nClearLength, nIdx;
MD5_CTX md5Ctx;
unsigned char salt[20];

for(nIdx=0; nIdx<8; nIdx++)
keyBuf[nIdx]=0;

*(int*)keyBuf=nNumObj;
strcpy(salt, "DgNab");
MD5_Init(&md5Ctx);
superSecret(salt, 5, &md5Ctx);
superSecret(keyBuf, 3, &md5Ctx);
superSecret(keyBuf+4, 2, &md5Ctx);
MD5_Final(rc4Key, &md5Ctx);
CreateRc4Sbox(rc4Key);
nClearLength=Rc4Decrypt(pCryptText, nCryptSize, pClearBuffer, nClearBufferSize, S);
return nClearLength;
}

/* The following is just some PDF parsing. The juicy part is over, I'm afraid... */

int removeSecurityString (unsigned char *pBeginBlock, unsigned char *pEndBlock)
{
unsigned char *pTemp;
for(pTemp=pBeginBlock; pTemp!=pEndBlock; pTemp++) {
if(!strncmp(pTemp, ENCRYPTION_STRING, strlen(ENCRYPTION_STRING))) {
while('\n'!=*pTemp && '\r'!=*pTemp) {
*pTemp=' ';
pTemp++;
}
}
}

return 0;
}

int scanFile
(const int fh, const int ofh, int *pnWrote)
{
int numObject, nCryptLen, nClearTextLength, bStayThere;
unsigned char *pStart, *pStartCrypted, *pBeginBlock, *pCurrent;

for(;;) {
bStayThere=1;
pCurrent=pObjBuf;
pCurrent+=strlen(OBJ_STR);
pStart=pCurrent;

while(bStayThere) {
if(1>read(fh,pCurrent,1)) {
removeSecurityString(pBeginBlock, pCurrent);
write(ofh, pBeginBlock, pCurrent-pBeginBlock);
return 0;
}

if(!strncmp(pCurrent-strlen(OBJ_STR), OBJ_STR, strlen(OBJ_STR))) {
pCurrent++;
write(ofh, pStart, pCurrent-pStart);
while(*pCurrent!=' ')
pCurrent--;
pCurrent--;
while(*pCurrent!=' ')
pCurrent--;
*pCurrent=0;
numObject=atoi(pStart);
bStayThere=0;
}
pCurrent++;
}

pCurrent=pObjBuf;
pCurrent+=strlen(END_OBJ_DELIM);
pBeginBlock=pCurrent;
bStayThere=1;
while(bStayThere) {
if(1>read(fh,pCurrent,1)) {
write(ofh, pBeginBlock, pCurrent-pBeginBlock);
return 0;
}

if(strncmp(pCurrent-strlen(END_OBJ_DELIM), END_OBJ_DELIM, strlen(END_OBJ_DELIM))) {
pCurrent++;
continue;
}

for(pStart=pObjBuf-strlen(END_OBJ_DELIM); pStart!=pCurrent ; pStart++ ) {
if(!bStayThere)
break;
if(strncmp(pStart, STREAM_STR, strlen(STREAM_STR)))
continue;
pStartCrypted=pStart+strlen(STREAM_STR)+2;
for(pStart=pStartCrypted; pStart!=pCurrent ; pStart++) {
if(strncmp(pStart, END_STREAM_STR, strlen(END_STREAM_STR)))
continue;
nCryptLen=pStart-pStartCrypted-1;
nClearTextLength=mainDecryptLoop(pStartCrypted,nCryptLen,pStartCrypted,nCryptLen,numObject);
bStayThere=0;
break;
}
}
write(ofh, pBeginBlock, 1+pCurrent-pBeginBlock);
bStayThere=0;
}
}
return 0;
}

int Usage
(
const char *szMessage
)
{
fprintf(stderr, "%s\n", szMessage);
fprintf(stderr, "Usage: \n");
fprintf(stderr, "-h : this message\n");
fprintf(stderr, "-i file : input file\n");
fprintf(stderr, "-o file : output file\n");
return 255;
}

int main (int argc, char **argv)
{
int ifh, ofh, nWrote, nOffs;
char *strInputFileName, *strOutputFileName, ch;
struct stat statStruct;
strInputFileName=NULL;
strOutputFileName=NULL;

while((ch=getopt(argc, argv, "hi:o:"))!= -1) {
switch(ch) {
case 'h':
return Usage("Help message requested");
break;
case 'i':
strInputFileName=optarg;
break;
case 'o':
strOutputFileName=optarg;
break;
}
}

if(NULL == strInputFileName)
return Usage("Must give input file");

if(NULL == strOutputFileName)
return Usage("Must give output file");

if(stat(strInputFileName, &statStruct))
return Usage("Error opening input file");

ifh=open(strInputFileName, O_RDONLY);
ofh=open(strOutputFileName, O_WRONLY|O_CREAT|O_TRUNC, 0666);

nWrote=0;
if(scanFile(ifh, ofh, &nWrote))
return Usage("Problems scanning file");

close(ifh);
close(ofh);
return 0;
}