10 from optparse
import OptionParser
18 def checkexist (path) :
19 if not os.path.exists(path) :
20 print(
'Error: file or dir "%s" is not exist.' % path)
27 def checkdir (path, force) :
28 if os.path.exists(path) :
29 if not os.path.isdir(path) :
30 print(
'Error: "%s" is not directory.' % path)
37 print(
'Error: could not create "%s".' % path)
40 print(
'Error: directory "%s" is not exist. create it or use -f option.' % path)
55 def k_scheduler(node, infile, outfile, indir, outdir, rsctime, mapper, reducer, multi, scrfile) :
61 template = open(
'kmrgenscript.template').read()
66 dir0 = os.path.dirname(os.path.realpath(__file__))
67 dir1 = os.path.realpath(dir0 +
'/../lib')
73 dir2 = os.path.realpath(kmrhome +
'/lib')
74 template = open(dir2 +
'/kmrgenscript.template').read()
78 print(
'Error: could not open job-script template.')
82 ncol = len(str(node -1))
86 stginstr +=
'#PJM --stgin "rank=* %s %%r:./"' \
87 % (
'./' + os.path.basename(reducer))
89 files = os.listdir(indir)
92 ipath = os.path.join(indir, file)
95 stginstr +=
'#PJM --stgin "rank=%s %s %%r:./work/"' % (rank, ipath)
102 ipath = os.path.join(indir, infile)
103 stginstr +=
'#PJM --stgin "rank=* %s%%0%sr %%r:./input"' % (ipath, ncol)
106 opath = os.path.join(outdir, outfile)
107 stgoutstr =
'#PJM --stgout "rank=* %%r:./output.%%r %s.%%0%sr"' % (opath, ncol)
112 execstr =
'mpiexec -n %s -of-proc output ./kmrshell -m %s ./input'\
113 % (node,
'./' + os.path.basename(mapper))
115 execstr =
'mpiexec -n %s -of-proc output ./kmrshell -m %s -r %s ./input'\
116 % (node,
'./' + os.path.basename(mapper),
'./' + os.path.basename(reducer))
119 execstr =
'mpiexec -n %s -of-proc output ./kmrshell -m %s ./work'\
120 % (node,
'./' + os.path.basename(mapper))
122 execstr =
'mpiexec -n %s -of-proc output ./kmrshell -m %s -r %s ./work'\
123 % (node,
'./' + os.path.basename(mapper),
'./' + os.path.basename(reducer))
127 script = template % {
'NODE': node,
'RSCTIME': rsctime,
'MAPPER': mapper,
'DATASTGIN': stginstr,
'DATASTGOUT': stgoutstr,
'EXEC': execstr,
'KMRHOME': kmrhome}
133 out = open(scrfile,
"w")
134 print(script, file=out)
150 def selectscheduler(node, infile, outfile, indir, outdir, rsctime, mapper, reducer, multi, sched, scrfile) :
152 k_scheduler(node, infile, outfile, indir, outdir, rsctime, mapper, reducer, multi, scrfile)
158 if __name__ ==
"__main__" :
160 usage =
"usage: %prog [options] -m mapper [-r reducer]" 161 parser = OptionParser(usage)
163 parser.add_option(
"-e",
164 "--number-of-exec-node",
167 help=
"number of execute node",
171 parser.add_option(
"-p",
172 "--input-file-prefix",
175 help=
"input filename prefix",
179 parser.add_option(
"-o",
183 help=
"output filename prefix",
187 parser.add_option(
"-d",
191 help=
"input directory",
195 parser.add_option(
"-O",
199 help=
"output directory",
203 parser.add_option(
"-t",
207 help=
"resource time",
211 parser.add_option(
"-m",
218 parser.add_option(
"-r",
225 parser.add_option(
"-S",
229 help=
"scheduler (default is 'K')",
233 parser.add_option(
"-w",
234 "--write-scriptfile",
237 help=
"script filename",
240 parser.add_option(
"-M",
244 help=
"multi input files to one node",
247 parser.add_option(
"-f",
254 (options, args) = parser.parse_args()
259 parser.error(
"missing parameter")
262 if not options.mapper :
263 print(
"mapper not specified\n")
266 checkexist(options.indir)
267 checkdir(options.outdir, options.force)
270 if options.indir ==
"./" :
271 print(
"-M option needs -d (input directory) option.\n")
273 files = os.listdir(options.indir)
274 if len(files) < options.node :
275 print(
'Node number is greater than number of files in %s.\n' % options.indir)
278 selectscheduler(options.node, options.infile, options.outfile,
279 options.indir, options.outdir, options.rsctime,
280 options.mapper, options.reducer, options.multi,
281 options.sched, options.scrfile)