/*******************************************************************************
 * Licensed Materials - Property of IBM
 * (c) Copyright IBM Corporation 2009. All Rights Reserved.
 * 
 * Note to U.S. Government Users Restricted Rights:  
 * Use, duplication or disclosure restricted by GSA ADP Schedule 
 * Contract with IBM Corp. 
 *******************************************************************************/
params = bsf.lookupBean('params')
acc = bsf.lookupBean('accumulator')
token = bsf.lookupBean('token')
tokenNum = bsf.lookupBean('tokenNum');

fileName = params.metadata.get('filename')
lang = params.detectedLanguage;
newTokenName = fileName + ':' + token.toString() + '_' + lang

importClass(Packages.com.ibm.supa.tokenizers.TTokenUtils)
newToken = TTokenUtils.newToken(newTokenName, token.getType())

// add old token followed by new token
// note that the new token has punctuation so it won't be
// searchable without fuzzy searches
// (since the query parser will toss out any punctuation)
acc.add(token)
acc.add(newToken)
alert(newToken.toString)