awk '$3 ~ /:0$/{print $4-1;next}{print $4}' test.txt
-0.238
0.754
0.848
0
0.851

awk '{if(substr($3,3)==0){$4=$4-1}else{$4=$4}}1' file

sdlcb@Goofy-Gen:~/AMD/SO$ cat file
1    1:?    1:0    0.762
2    1:?    2:1    0.754
3    1:?    2:1    0.848
4    1:?    1:0    1
5    1:?    2:1    0.851
sdlcb@Goofy-Gen:~/AMD/SO$ awk '{if(substr($3,3)==0){$4=$4-1}else{$4=$4}}1' file
1 1:? 1:0 -0.238
2 1:? 2:1 0.754
3 1:? 2:1 0.848
4 1:? 1:0 0
5 1:? 2:1 0.851

$ awk '{split($3,a,":");if(a[2]==0){$4=$4-1}}1' file | column -t
1  1:?  1:0  -0.238
2  1:?  2:1  0.754
3  1:?  2:1  0.848
4  1:?  1:0  0
5  1:?  2:1  0.851

awk 'split($3,a,":"){print a[2]?$4-1:$4}' file

awk 'split($3,a,":"){$4=a[2]?$4-1:$4}' file