greatly differing processing time between java and Linux while calculating hashes?
 
~ 
 I have been noticing great differences (> 25%) in the processing time of
hashes between java and the implementations in the OS, which cannot attributed
to block size (I played with it as well). What I consistently got was the java
is much faster for sha512 and sha384, but then for sha256, sha1 and md5 Linux
becomes then much faster ... 
~ 
 So I grabbed two relatively large media files from youtube:
~ 
$ ls -l DQfUaXLk_sw.mp4
-rw-r--r-- 1 knoppix knoppix 628588285 Apr  1 18:09 DQfUaXLk_sw.mp4
$ ls -l 0buBJlPo9us.flv
-rw-r--r-- 1 knoppix knoppix 475965918 Aug 31  2010 0buBJlPo9us.flv
~ 
 Quickly coded some test:
~ 
import java.util.Iterator;
import java.util.Set;
import java.util.HashSet;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.security.MessageDigest;
import java.security.Security;
import java.security.Provider;
import java.security.NoSuchAlgorithmException;
// __  
public class CheckSum00Test{
// __ 
 private static final String aLnSep = System.getProperty("line.separator");
// __  
 private static final int _BLCK_SZ = 512;
// __ 
 private static final String[] aSecKeys = new String[]{
  "MessageDigest.", "Alg.Alias.MessageDigest."
 };
// __ 
 private static final String[] aSecPrvdrs = getProviders();
// __ 
 private static final String[] getProviders() throws SecurityException{
  String[] aArSec = null; 
// __  
  Set<String> SSPrvdrs = new HashSet<String>();
  boolean IsPrvdr;
  Iterator Itr;
  int iPrvdrs = Integer.MIN_VALUE, iPrvdrPrfxL = aSecKeys.length, iSSz;
  String aSecKey;
  Provider[] SecPrvdr = Security.getProviders();
  boolean IsSystemPrvdrs = ((SecPrvdr != null) && 
((iPrvdrs = SecPrvdr.length) > 0));
  if(IsSystemPrvdrs){
   for(int i = 0; (i < iPrvdrs); ++i){
    Itr = SecPrvdr[i].keySet().iterator();
    while(Itr.hasNext()){
     String aKey = (String)Itr.next();
     IsPrvdr = false;
     for(int j = 0; (j < iPrvdrPrfxL) && !IsPrvdr; ++j){
      IsPrvdr = aKey.startsWith(aSecKeys[j]);
// __ 
      if(IsPrvdr){
       aSecKey = aKey.split(" ")[0];
       if(aSecKey != null){
        aSecKey = aSecKey.trim();
        aSecKey = aSecKey.substring(aSecKeys[j].length());
        if(aSecKey.length() > 0){ SSPrvdrs.add(aSecKey); }
       }// (aSecKey != null)
      }// (IsPrvdr)
     }// j [0, iPrvdrPrfxL)
    }// (Itr.hasNext()){
   }// i [0, iPrvdrs)
// __ 
   IsSystemPrvdrs = ((iSSz = SSPrvdrs.size()) > 0);
   if(IsSystemPrvdrs){
    aArSec = new String[iSSz];
    SSPrvdrs.toArray(aArSec); 
   }
  }
// __ 
  if(!IsSystemPrvdrs){
   String aX = "FATAL ERROR: java implementation does NOT seem to 
include Security Providers!" + aLnSep;
   aX += "// __ java.version: " + System.getProperty("java.version") + aLnSep;
// ... 
   aX += "~";
   throw new SecurityException(aLnSep + "// __ " + aX + aLnSep);
  }
// __  
  return(aArSec);
 }
// __  
 private static String getErrSecAlgos(){
  StringBuilder aBldr = new StringBuilder();
  aBldr.append(aLnSep + "// __ The " + aSecPrvdrs.length + " checksum 
algorithms that could be used are: " + aLnSep + aLnSep);
  for(int i = 0; (i < aSecPrvdrs.length); ++i){ aBldr.append((i + 1) + ":  "
+ aSecPrvdrs[i] + aLnSep); }
  return(aBldr.toString());
 }
// __  
 public static void main(String[] args){
  String aKNm = "CheckSum00Test"; 
  if((args != null) && (args.length == 2)){
   try{
    long lTm00 = System.currentTimeMillis();
// __ 
    MessageDigest md = MessageDigest.getInstance(args[0]);
// __ 
    FileInputStream fis = new FileInputStream(args[1]);
    byte[] dataBytes = new byte[_BLCK_SZ];
    int nread = 0; 
    while ((nread = fis.read(dataBytes)) != -1) { md.update(dataBytes,
0, nread); };
    fis.close();
    byte[] mdbytes = md.digest();
    long lTm02 = System.currentTimeMillis();
// __ byte2hex
    StringBuilder aBldr = new StringBuilder();
    for (int i = 0; i < mdbytes.length; i++) {
     aBldr.append(Integer.toString((mdbytes[i] & 0xff) + 0x100, 16)
..substring(1));
    }
    System.err.println("// __ " + args[0] + " encrypting  \"" + args[1]
+ "\":\"" + aBldr.toString()  + "\"  took: " +(lTm02 - lTm00) + " (ms)");
   }catch(FileNotFoundException FlNtFndX){ FlNtFndX.
printStackTrace(System.err); }
     catch(IOException IOX){ IOX.printStackTrace(System.err); }
    catch(NoSuchAlgorithmException NSekAlgoX){
//     System.err.println(getErrSecAlgos());
     NSekAlgoX.printStackTrace(System.err);
    }
  }// ((args != null) && (args.length == 2))
  else{
   System.err.println(aLnSep + "// __ usage: java " + aKNm + " 
<sum algorithm> <input file>" + aLnSep);
   System.err.println(getErrSecAlgos());
  }
 }
}
~ 
 then I went:
~ 
date; time  java CheckSum00Test SHA-512 <input_file>; date;
date; time sha512sum -b <input_file>; date;
date; time  java CheckSum00Test SHA-384 <input_file>; date;
date; time sha384sum -b <input_file>; date;
date; time java CheckSum00Test SHA-256 <input_file>; date;
date; time sha256sum -b <input_file>; date;
date; time java CheckSum00Test SHA-1 <input_file>; date;
date; time sha1sum -b <input_file>; date;
date; time  java CheckSum00Test MD5 <input_file>; date;
date; time md5sum -b <input_file>; date;
~ 
 What do you think is going on here?
~ 
 thanks
 lbrtchx