Category: Programming

RServe can get all variables from the ls() command

This is a working example in mac to get all variables from R’s ls() command using RServe

package com.mycompany.mmm;

import java.io.*;
import org.rosuda.REngine.REXP;
import org.rosuda.REngine.Rserve.RConnection;

/**
 * helper class that consumes output of a process. In addition, it filter output of the REG command on Windows to look for InstallPath registry entry which specifies the location of R.
 */
class StreamHog extends Thread {

	InputStream is;
	boolean capture;
	String installPath;

	StreamHog(InputStream is, boolean capture) {
		this.is = is;
		this.capture = capture;
		start();
	}

	public String getInstallPath() {
		return installPath;
	}

	public void run() {
		try {
			BufferedReader br = new BufferedReader(new InputStreamReader(is));
			String line = null;
			System.out.println("started");
			while ((line = br.readLine()) != null) {
//				if (capture) { // we are supposed to capture the output from REG command
//					int i = line.indexOf("InstallPath");
//					if (i >= 0) {
//						String s = line.substring(i + 11).trim();
//						int j = s.indexOf("REG_SZ");
//						if (j >= 0) {
//							s = s.substring(j + 6).trim();
//						}
//						installPath = s;
//						System.out.println("R InstallPath = " + s);
//					}
//				} else {
//					System.out.println("Rserve>" + line);
//				}
				System.out.println("Rserve>" + line);
			}
		} catch (IOException e) {
			e.printStackTrace();
		}
	}
}

/**
 * simple class that start Rserve locally if it's not running already - see mainly <code>checkLocalRserve</code> method. It spits out quite some debugging outout of the console, so feel free to modify
 * it for your application if desired.<p>
 * <i>Important:</i> All applications should shutdown every Rserve that they started! Never leave Rserve running if you started it after your application quits since it may pose a security risk.
 * Inform the user if you started an Rserve instance.
 */
public class StartRserve {

	/**
	 * shortcut to <code>launchRserve(cmd, "--no-save --slave", "--no-save --slave", false)</code>
	 */
	public static boolean launchRserve(String cmd) {
		return launchRserve(cmd, "--no-save --slave", "--no-save --slave", false);
	}

	/**
	 * attempt to start Rserve. Note: parameters are <b>not</b> quoted, so avoid using any quotes in arguments
	 *
	 * @param cmd command necessary to start R
	 * @param rargs arguments are are to be passed to R
	 * @param rsrvargs arguments to be passed to Rserve
	 * @return <code>true</code> if Rserve is running or was successfully started, <code>false</code> otherwise.
	 */
	public static boolean launchRserve(String cmd, String rargs, String rsrvargs, boolean debug) {
		try {
			Process p;
			boolean isWindows = false;
			String osname = System.getProperty("os.name");
			if (osname != null && osname.length() >= 7 && osname.substring(0, 7).equals("Windows")) {
				isWindows = true;
				/* Windows startup */
				p = Runtime.getRuntime().exec("\"" + cmd + "\" -e \"library(Rserve);Rserve(" + (debug ? "TRUE" : "FALSE") + ",args='" + rsrvargs + "')\" " + rargs);
			} else /* unix startup */ {
				p = Runtime.getRuntime().exec(new String[]{
					"/bin/sh", "-c",
					"echo 'library(Rserve);Rserve(" + (debug ? "TRUE" : "FALSE") + ",args=\"" + rsrvargs + "\")'|" + cmd + " " + rargs
				});
//				p = Runtime.getRuntime().exec("/Users/peter/Library/R/3.4/library/Rserve/libs//Rserve --no-save --slave");
			}
			System.out.println("waiting for Rserve to start ... (" + p + ")");
			// we need to fetch the output - some platforms will die if you don't ...
			StreamHog errorHog = new StreamHog(p.getErrorStream(), false);
			StreamHog outputHog = new StreamHog(p.getInputStream(), false);
			if (!isWindows) /* on Windows the process will never return, so we cannot wait */ {
				p.waitFor();
			}
			System.out.println("call terminated, let us try to connect ...");
		} catch (Exception x) {
			System.out.println("failed to start Rserve process with " + x.getMessage());
			return false;
		}
		int attempts = 5;
		/* try up to 5 times before giving up. We can be conservative here, because at this point the process execution itself was successful and the start up is usually asynchronous */
		while (attempts > 0) {
			try {
				RConnection c = new RConnection();
				System.out.println("Rserve is running.");
				c.close();
				return true;
			} catch (Exception e2) {
				System.out.println("Try failed with: " + e2.getMessage());
			}
			/* a safety sleep just in case the start up is delayed or asynchronous */
			try {
				Thread.sleep(500);
			} catch (InterruptedException ix) {
			};
			attempts--;
		}
		return false;
	}

	/**
	 * checks whether Rserve is running and if that's not the case it attempts to start it using the defaults for the platform where it is run on. This method is meant to be set-and-forget and cover
	 * most default setups. For special setups you may get more control over R with <<code>launchRserve</code> instead.
	 */
	public static boolean checkLocalRserve() {
		if (isRserveRunning()) {
			return true;
		}
		String osname = System.getProperty("os.name");
		if (osname != null && osname.length() >= 7 && osname.substring(0, 7).equals("Windows")) {
			System.out.println("Windows: query registry to find where R is installed ...");
			String installPath = null;
			try {
				Process rp = Runtime.getRuntime().exec("reg query HKLM\\Software\\R-core\\R");
				StreamHog regHog = new StreamHog(rp.getInputStream(), true);
				rp.waitFor();
				regHog.join();
				installPath = regHog.getInstallPath();
			} catch (Exception rge) {
				System.out.println("ERROR: unable to run REG to find the location of R: " + rge);
				return false;
			}
			if (installPath == null) {
				System.out.println("ERROR: canot find path to R. Make sure reg is available and R was installed with registry settings.");
				return false;
			}
			return launchRserve(installPath + "\\bin\\R.exe");
		}
		return launchRserve("/opt/local/Library/Frameworks/R.framework/Versions/3.4/Resources/bin/R");
	}

	/**
	 * check whether Rserve is currently running (on local machine and default port).
	 *
	 * @return <code>true</code> if local Rserve instance is running, <code>false</code> otherwise
	 */
	public static boolean isRserveRunning() {
		try {
			RConnection c = new RConnection();
			System.out.println("Rserve is running.");
			c.close();
			return true;
		} catch (Exception e) {
			System.out.println("First connect try failed with: " + e.getMessage());
		}
		return false;
	}

	/**
	 * just a demo main method which starts Rserve and shuts it down again
	 */
	public static void main(String[] args) {
		System.out.println("checkLocalRserve()=" + checkLocalRserve());
		try {
			RConnection c = new RConnection();
			REXP x;
			x = c.eval("ls()");
			x = c.eval("a=c(1:10)");
			x = c.eval("bcd=rep(a,3)");
			System.out.println("a="+x.asString());
			x = c.eval("ls()");
			x = c.eval("R.version.string");
			System.out.println(x.asString());
			c.shutdown();
			System.out.println("end");
		} catch (Exception x) {
		}
	}
}

read count : 2

Netbeans R plugins with Rserve

Netbeans R plugins with Rserve. Rserve can interact with R using Java, huge ability to hock things in R. Heading to the first release of the R plugin. Anyone want to join the team, email me, Peter (mcheung63@hotmail.com).

Below are the first release features

  1. Syntax highlight, done, i think at least better than R-studio
  2. Function parameters popip, R-studio is totally useless at this point, it show up all R functions instead of the parameter list of the current function.
  3. R console, half done. I was using pipeline to create a R shell. Changing to us Rserve.
  4. Plot view. I am doing this
  5. Help/History window. Not start yet.
  6. Environment to see current defined variables. Not start yet.

The goals:

  1. Create a complete environment for people to run and debug R
  2. Add profiling feature, display performance in each line
  3. Compete with R-studio, get more users to use Netbeans. R-studio is simple and lots of places we can do better.
  4. Add support for Clouds. AWS/Azure
  5. Add support for frameworks, Hadoop/Spark
Syntax highlight is more nice than R-studio

read count : 321

Correct command to compile R 3.4.0 in Mac

Correct command to compile R 3.4.0 in Mac, i steal this commands from macports by “sudo port -d configure R”, so i know the default gfortran (/usr/local/bin/gfortran) won’t work, i have to use /opt/local/bin/gfortran-mp-6. Below is the minimal parameters set to build R.

LDFLAGS='-L/opt/local/lib -Wl,-headerpad_max_install_names' CPPFLAGS='-I/opt/local/include' CPATH='/opt/local/include' CXX='/usr/bin/clang++' CC='/usr/bin/clang' F77='/opt/local/bin/gfortran-mp-6' ../R-3.4.0/configure --prefix=/Users/peter/Desktop/install-r

read count : 13

Learning how to do raw logic programming

Learning how to do raw logic programming, convert the programming logic directly into logic gates. Found these two book in central library. The first one teach you everything you need to build up logic design. In undergrade year one, i remember uni was teaching us k-map and state diagram but without telling us why they are important and the use-case. I totally get messed and lost in space. Now I finally know how to use them. The verilog book  teach you how to convert programming looping into logic gates, but this book has too much text.

read count : 14

[Solved] Setup sharepoint 2016 standalone server configuration fail

Setup sharepoint 2016 standalone fail, the configuration wizard said “the specified user is a local account local accounts should only be used in standalone mode”, below command can fix it

New-SPConfigurationDatabase –DatabaseName SharePoint_Config –DatabaseServer <server> –AdministrationContentDatabaseName SharePoint_Content –Passphrase (ConvertTo-SecureString <password> –AsPlaintext –Force) –FarmCredentials (Get-Credential) -localserverrole SingleServerFarm

Make sure you has no error from the above command before you run the configuration wizard again, select “connect to farm” to finish the setup of sp2016

read count : 18

R code runs slowly

I try use R to find out which two stocks run very similarly in last 30 days. My database is MySQL. but it runs slowly. I read the records for each stock in a for loop and append it to a data frame. After that i use a double for loop to find out the maximum value of cor() to those data.

library(RMySQL)
library(quantmod)

killDbConnections <- function () {
  all_cons <- dbListConnections(MySQL())
  #print(all_cons)
  for(con in all_cons)
    +  dbDisconnect(con)
  #print(paste(length(all_cons), " connections killed."))
}

killDbConnections()

mydb = dbConnect(MySQL(), user='stockmonitor', password='stockmonitor', dbname='stockmonitor', host='127.0.0.1')

getStocks <- function(stockNo){
  rs=dbSendQuery(mydb, "select distinct date(date) from recordDaily order by date(date) limit 0,30")
  data=fetch(rs, n=-1)
  str=""
  for (i in 1:(nrow(data)-1)){
    if (str==""){
      str=paste0(str, "select '", data[i,], "' as date union")
    }else{
      str=paste0(str, " select '", data[i,], "' union")
    }
  }
  str=paste0(str, " select '", data[nrow(data),], "'")
  str=paste0("select a.date, IFNULL(b.price, -1) from (\n", str, ") as a
             left join 
             (select * from recordDaily where stockNo='",stockNo,"') as b 
             on a.date=date(b.date)")
  
  rs=dbSendQuery(mydb, str)
  data=fetch(rs, n=-1)
  return(data)
}

appendStock <- function(df, stockNo){
  stockData=getStocks(stockNo)
  df=data.frame(rbind(as.matrix(df), as.matrix(t(c(stockNo,stockData[,2])))))
  return(df)
}

#colClasses = c("character", rep("numeric", length(data[,1])))

rs=dbSendQuery(mydb, "select distinct date(date) from recordDaily order by date(date) limit 0,30")
data=fetch(rs, n=-1)
col.names = c("Stock", data[,1])
df <- read.table(text = "", col.names = col.names)

rs=dbSendQuery(mydb, "select distinct stockNo from recordDaily order by stockNo")
data=fetch(rs, n=-1)
for (i in 1:nrow(data)){
  print(paste0('appending ', data[i,]))
  df=appendStock(df, data[i,])
}

maxCor=-999999
name1=""
name2=""
for (x in 1:length(df[,1])){
  for (y in 1:length(df[,1])){
    if (x!=y){
      c=cor(as.numeric(as.matrix(df[x,-1])), as.numeric(as.matrix(df[y,-1])))
      print(paste0("cor=",c, "=", df[x,1],",",df[y,1]))
      if (abs(c)>maxCor){
        maxCor=abs(c)
        name1=df[x,1]
        name2=df[y,1]
        print(paste0(name1,", ", name2))
      }
    }
  }
}

print(paste0(cor,"=",name1,",",name2))

read count : 6