132 lines
3.5 KiB
Python
132 lines
3.5 KiB
Python
#!/usr/bin/env python
|
|
|
|
|
|
import os
|
|
import re
|
|
import time
|
|
import random
|
|
|
|
dataset = "RTE2_dev"
|
|
score = 0.0
|
|
minScore = 0.60
|
|
|
|
memory = "-ms2g -mx7g"
|
|
|
|
|
|
# Set up file paths -------------------------------------------------------------
|
|
|
|
dataDir = "/u/nlp/rte/data/byformat"
|
|
tmpDir = "/tmp/rte-regression.%d" % os.getpid()
|
|
os.mkdir(tmpDir)
|
|
rteFile = "%s/rte/%s.xml" % (dataDir, dataset)
|
|
kbeFile = "%s/%s.kbe.xml" % (tmpDir, dataset)
|
|
pipelineFile = "%s/%s.pipeline.info.xml" % (tmpDir, dataset)
|
|
alignFile = "%s/%s.align.xml" % (tmpDir, dataset)
|
|
logFile = "%s/%s.log" % (tmpDir, dataset)
|
|
regressionFile = "%s/regression/%s.regression.log" % (dataDir, dataset)
|
|
|
|
|
|
# Make KBE file from RTE file ---------------------------------------------------
|
|
|
|
def makeKBEFile():
|
|
javaclass = "edu.stanford.nlp.util.XMLTransformer"
|
|
xsltransformer = "/u/nlp/rte/data/resources/RTE_to_KBEval.xsl"
|
|
cmd = "java -server %s %s " % (memory, javaclass) + \
|
|
"-in %s " % rteFile + \
|
|
"-out %s " % kbeFile + \
|
|
"-transform %s " % xsltransformer + \
|
|
"> %s 2>&1 " % logFile
|
|
# print "cmd is:\n", cmd
|
|
os.system(cmd)
|
|
|
|
|
|
# Annotation --------------------------------------------------------------------
|
|
|
|
def doAnnotation():
|
|
javaclass = "edu.stanford.nlp.rte.RTEPipeline"
|
|
cmd = "java -server %s %s " % (memory, javaclass) + \
|
|
"-kbeIn %s " % kbeFile + \
|
|
"-infoOut %s " % pipelineFile + \
|
|
"> %s 2>&1 " % logFile
|
|
# print "cmd is:\n", cmd
|
|
os.system(cmd)
|
|
|
|
|
|
# Alignment & inference ---------------------------------------------------------
|
|
|
|
def doAlignmentAndInference():
|
|
aligner = "stochastic"
|
|
javaclass = "edu.stanford.nlp.rte.KBETester"
|
|
cmd = "java -server %s %s " % (memory, javaclass) + \
|
|
"-info %s " % pipelineFile + \
|
|
"-saveAlignments %s " % alignFile + \
|
|
"-aligner %s " % aligner + \
|
|
"-twoClass " + \
|
|
"-balancedData " + \
|
|
"-verbose 1 " + \
|
|
"> %s 2>&1 " % logFile
|
|
# print "cmd is:\n", cmd
|
|
os.system(cmd)
|
|
|
|
|
|
# Extract score -----------------------------------------------------------------
|
|
|
|
def extractScore():
|
|
for line in os.popen("grep '^Accuracy:' %s" % logFile):
|
|
line = line.strip()
|
|
# print line
|
|
fields = re.split('\s+', line)
|
|
score = float(fields[-1])
|
|
return score
|
|
|
|
|
|
# Get previous score ------------------------------------------------------------
|
|
|
|
def getPreviousScore():
|
|
prev = 0.0
|
|
for line in os.popen("grep '^PASS' %s" % regressionFile):
|
|
line = line.strip()
|
|
# print line
|
|
fields = re.split('\s+', line)
|
|
prev = float(fields[1])
|
|
return prev
|
|
|
|
|
|
# Save score --------------------------------------------------------------------
|
|
|
|
def saveScore(score, minScore, logFile):
|
|
if score >= minScore:
|
|
result = "PASS"
|
|
else:
|
|
result = "FAIL"
|
|
f = open(regressionFile, "a")
|
|
print >>f, \
|
|
"%s %.4f %.4f %s %s" % \
|
|
(result,
|
|
score,
|
|
minScore,
|
|
time.strftime("%Y%m%d-%H%M%S"),
|
|
logFile)
|
|
f.close()
|
|
|
|
|
|
# main --------------------------------------------------------------------------
|
|
|
|
makeKBEFile()
|
|
doAnnotation()
|
|
doAlignmentAndInference()
|
|
# score = random.random()
|
|
score = extractScore()
|
|
minScore = max(minScore, getPreviousScore())
|
|
|
|
if score >= minScore:
|
|
print "PASS score %.4f >= min %.4f" % (score, minScore)
|
|
else:
|
|
print "FAIL score %.4f >= min %.4f, output in %s" % (score, minScore, logFile)
|
|
|
|
saveScore(score, minScore, logFile)
|
|
|
|
|
|
|
|
|